diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa7e75cea..c40694e25 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: check - args: "--features decimal,metadata,serde,debugging" + args: "--features decimal,metadata,serde,debugging,grain" # typical build with various feature combinations build: @@ -46,6 +46,7 @@ jobs: os: [ubuntu-latest] flags: - "" + - "--features grain" - "--features testing-environ,debugging" - "--features testing-environ,metadata" - "--features testing-environ,serde" @@ -62,6 +63,7 @@ jobs: - "--features testing-environ,f32_float,serde,metadata,internals,debugging" - "--features testing-environ,no_custom_syntax,serde,metadata,internals,debugging" - "--tests --features testing-environ,only_i32,serde,metadata,internals,debugging" + - "--tests --features testing-environ,only_i32,serde,metadata,internals,debugging,grain" - "--features testing-environ,only_i64,serde,metadata,internals,debugging" - "--features testing-environ,no_index,serde,metadata,internals,debugging" - "--features testing-environ,no_object,serde,metadata,internals,debugging" @@ -107,6 +109,7 @@ jobs: - {os: ubuntu-latest, flags: "--profile unix", experimental: false} - {os: windows-latest, flags: "--profile windows", experimental: true} - {os: macos-latest, flags: "--profile macos", experimental: false} + - {os: ubuntu-latest, flags: "--profile unix --features rhai/grain", experimental: false} steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 77c1ef44e..d7b55da02 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ doc/rhai.json .idea .idea/* src/eval/chaining.rs +.zed/ diff --git a/Cargo.toml b/Cargo.toml index a0e1f789e..082550a10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = [".", "codegen", "codegen/tests/custom_root"] name = "rhai" version = "1.25.1" rust-version = "1.66.0" -edition = "2018" +edition = "2021" resolver = "2" authors = ["Jonathan Turner", "Lukáš Hozda", "Stephen Chung", "jhwgh1968"] description = "Embedded scripting for Rust" @@ -70,7 +70,9 @@ debugging = ["internals"] ## Features and dependencies required by `bin` tools: `decimal`, `metadata`, `serde`, `debugging` and [`rustyline`](https://crates.io/crates/rustyline). bin-features = ["decimal", "metadata", "serde", "debugging", "rustyline"] ## Enable fuzzing via the [`arbitrary`](https://crates.io/crates/arbitrary) crate. -fuzz = ["arbitrary", "rust_decimal?/rust-fuzz", "serde"] +fuzz = ["arbitrary", "rust_decimal?/rust-fuzz", "serde", "grain"] +## Enable the experimental `grain` bytecode VM +grain = [] #! ### System Configuration Features @@ -147,6 +149,27 @@ required-features = ["serde"] name = "definitions" required-features = ["metadata", "internals"] +# VM-versus-walker timings. Run with `--release`; `-- --check` exits non-zero +# on a case that has fallen below the floor recorded beside it. +[[example]] +name = "grain_bench" +required-features = ["grain"] + +# The grain harnesses, as one binary. `tests/mod.rs` wraps them in an inline +# `mod grain`, which is what puts their module paths inside `tests/grain/`. +[[test]] +name = "grain" +path = "tests/mod.rs" +required-features = ["grain"] + +# Its own binary, and not negotiable: it installs a counting global allocator +# and reads process-global counters as deltas around each call, so any test +# allocating on another thread corrupts every number it reports. +[[test]] +name = "grain_allocation_efficiency" +path = "tests/grain/allocation.rs" +required-features = ["grain"] + [profile.release] lto = "fat" codegen-units = 1 diff --git a/examples/grain_bench.rs b/examples/grain_bench.rs new file mode 100644 index 000000000..d204afb85 --- /dev/null +++ b/examples/grain_bench.rs @@ -0,0 +1,261 @@ +//! Rough VM-versus-walker timings, on the same AST. +//! +//! Indicative, not criterion: repeated runs, reporting the fastest of each and +//! the spread around it so a single scheduling hiccup does not read as a +//! result. Run with `--release`; a debug build measures bounds checks more than +//! anything else. +//! +//! # Catching a regression +//! +//! `cargo run --release --example bench -- --check` exits non-zero if any case +//! has fallen below the floor recorded beside it. +//! +//! What is compared is the **ratio**, not the time. Absolute milliseconds say +//! as much about the machine as about the code, and there is no useful way to +//! commit one; the walker and the VM run back to back on the same machine in +//! the same process, so their ratio mostly divides the machine out. Mostly, not +//! entirely — cache size and core count still move it — so a floor sits about +//! 15% under the observed figure. That is wide enough not to cry wolf on a +//! slower runner and tight enough that losing a fast path shows up. + +use std::time::{Duration, Instant}; + +use rhai::grain::{Compiler, Program, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +const RUNS: usize = 9; + +struct Case { + name: &'static str, + source: &'static str, + iterations: usize, + /// Whether the run needs the callback wrappers installed, which costs an + /// owned program and a module built per run. + callbacks: bool, + /// The speedup this case must not drop below. + /// + /// Beside the source rather than in a table of its own, so changing one + /// without the other is visible in the diff. + floor: f64, +} + +/// The fastest sample, and how much slower the middle one was. +/// +/// Noise on a timing is one-sided — nothing makes a run finish sooner than it +/// can — so the fastest sample is the least contaminated estimate, and the +/// median is here only to say how contaminated the rest were. A wide spread +/// means the number below it should not be read closely. +struct Timing { + fastest: Duration, + median: Duration, +} + +impl Timing { + fn secs(&self) -> f64 { + self.fastest.as_secs_f64() + } + + /// How far the median sits above the fastest, as a fraction. + fn spread(&self) -> f64 { + self.median.as_secs_f64() / self.fastest.as_secs_f64() - 1.0 + } +} + +const CASES: &[Case] = &[ + Case { + name: "tight integer loop", + source: "let s = 0; let i = 0; while i < 20000 { s += i; i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.30, + }, + Case { + name: "float arithmetic", + source: "let x = 0.0; let i = 0; while i < 20000 { x += (i.to_float() * 1.5) / 2.5; i += 1; } x", + iterations: 20, + callbacks: false, + floor: 1.10, + }, + Case { + name: "script fn calls", + source: "fn add(a, b) { a + b } let s = 0; let i = 0; while i < 5000 { s = add(s, i); i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.55, + }, + // The VM scans its case hashes; rhai probes a hash map. Two sizes, + // because which of those wins is a question about how many arms there + // are, and a `switch` nobody would write is the only place the scan can + // lose. + Case { + name: "switch, 4 arms", + source: "let s = 0; let i = 0; while i < 20000 { \ + switch i % 4 { 0 => s += 1, 1 => s += 2, 2 => s += 3, _ => s += 4 } \ + i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.40, + }, + Case { + name: "switch, 16 arms", + source: "let s = 0; let i = 0; while i < 20000 { \ + switch i % 16 { \ + 0 => s += 1, 1 => s += 2, 2 => s += 3, 3 => s += 4, \ + 4 => s += 5, 5 => s += 6, 6 => s += 7, 7 => s += 8, \ + 8 => s += 9, 9 => s += 10, 10 => s += 11, 11 => s += 12, \ + 12 => s += 13, 13 => s += 14, 14 => s += 15, _ => s += 16 } \ + i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.35, + }, + Case { + name: "branch heavy", + source: "let s = 0; let i = 0; while i < 20000 { if i % 3 == 0 { s += 1; } else if i % 3 == 1 { s += 2; } else { s -= 1; } i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.45, + }, + // The one case the VM is expected to lose. Every element is a boundary out + // of the VM, through rhai's dispatch and back into a second `Vm` with an + // empty resolution cache — where the walker stays inside itself and reaches + // the closure body directly. 1000 crossings per iteration. + Case { + name: "native callbacks", + source: "let a = []; let i = 0; while i < 500 { a.push(i); i += 1; } \ + let b = a.map(|x| x * 2); b.filter(|x| x % 3 == 0).len()", + iterations: 20, + callbacks: true, + floor: 0.25, + }, +]; + +fn time(mut run: impl FnMut()) -> Timing { + let mut samples: Vec = (0..RUNS) + .map(|_| { + let start = Instant::now(); + run(); + start.elapsed() + }) + .collect(); + samples.sort_unstable(); + Timing { + fastest: samples[0], + median: samples[RUNS / 2], + } +} + +fn main() { + let check = std::env::args().any(|arg| arg == "--check"); + let engine = Engine::new(); + + // Rhai's default options include FAST_OPS, which makes the walker + // short-circuit binary operators and op-assignments straight to builtin + // function pointers — no hash, no resolution cache + // (`func/call.rs:1775-1799`, `eval/stmt.rs:131-148`). Turning it off + // measures how much of the walker's speed comes from that, and therefore + // how much of the VM's planned "typed fast opcodes" win is already taken. + let mut slow_engine = Engine::new(); + slow_engine.set_fast_operators(false); + + println!( + "{:<22} {:>11} {:>11} {:>9} {:>7} {:>8} {:>11} {:>10}", + "", "walker", "vm", "speedup", "floor", "spread", "walker-slow", "fragments" + ); + + let mut below_floor = Vec::new(); + + for case in CASES { + let ast = engine.compile(case.source).expect("must compile"); + let program: Program = Compiler::new().compile(&ast); + // Owned and shared only where a pointer can escape to a native, so the + // ordinary cases keep measuring the ordinary path. + let shared = case + .callbacks + .then(|| Compiler::new().compile(&ast).into_shared()); + let run_vm = || match &shared { + Some(shared) => Vm::new(&engine).eval_with_callbacks(&mut Scope::new(), shared), + None => Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program), + }; + + // Same result, or the comparison is meaningless. + let expected = engine + .eval_ast_with_scope::(&mut Scope::new(), &ast) + .expect("walker must succeed"); + let actual = run_vm().expect("vm must succeed"); + assert_eq!( + format!("{expected:?}"), + format!("{actual:?}"), + "{} disagreed, so its timing means nothing", + case.name, + ); + + let walker = time(|| { + for _ in 0..case.iterations { + let _ = engine + .eval_ast_with_scope::(&mut Scope::new(), &ast) + .unwrap(); + } + }); + + let vm = time(|| { + for _ in 0..case.iterations { + let _ = run_vm().unwrap(); + } + }); + + let slow_ast = slow_engine.compile(case.source).expect("must compile"); + let walker_slow = time(|| { + for _ in 0..case.iterations { + let _ = slow_engine + .eval_ast_with_scope::(&mut Scope::new(), &slow_ast) + .unwrap(); + } + }); + + // The spread reported is the VM's, because that is the number the + // floor is about. A walker sample knocked sideways shows up in the + // speedup anyway. + let speedup = walker.secs() / vm.secs(); + println!( + "{:<22} {:>9.1}ms {:>9.1}ms {:>8.2}x {:>6.2}x {:>7.0}% {:>9.1}ms {:>10}", + case.name, + walker.secs() * 1000.0, + vm.secs() * 1000.0, + speedup, + case.floor, + vm.spread() * 100.0, + walker_slow.secs() * 1000.0, + program.residual_nodes(), + ); + + if speedup < case.floor { + below_floor.push(format!( + "\n {}: {speedup:.2}x, floor {:.2}x (VM samples spread {:.0}%)", + case.name, + case.floor, + vm.spread() * 100.0, + )); + } + } + + if below_floor.is_empty() { + return; + } + + // Printed whether or not this is a gated run: a regression is worth seeing + // even when nobody asked for an exit code. + eprintln!( + "\n{} case(s) below their floor:{}", + below_floor.len(), + below_floor.join(""), + ); + eprintln!( + "\nA wide spread means the machine was busy — rerun before believing it. \ + If the loss is real, either find it or move the floor in the same commit \ + that causes it.", + ); + if check { + std::process::exit(1); + } +} diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index e7d47fb0f..1f8e311ac 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true anyhow = "1.0.78" arbitrary = { version = "1.3.2", features = ["derive"] } libfuzzer-sys = "0.4" -rhai = { path = "..", features = ["fuzz", "decimal", "metadata", "debugging"] } +rhai = { path = "..", features = ["fuzz", "decimal", "metadata", "debugging", "grain"] } serde = { version = "1.0.194", features = ["derive"] } # Prevent this from interfering with workspaces @@ -19,7 +19,9 @@ serde = { version = "1.0.194", features = ["derive"] } members = ["."] [profile.release] -debug = 1 +debug = true +debug-assertions = true +overflow-checks = true [[bin]] name = "scripting" @@ -38,3 +40,24 @@ name = "fuzz_serde" path = "fuzz_targets/fuzz_serde.rs" test = false doc = false + +[[bin]] +name = "load" +path = "fuzz_targets/load.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "roundtrip" +path = "fuzz_targets/roundtrip.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "generated" +path = "fuzz_targets/generated.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/generated.rs b/fuzz/fuzz_targets/generated.rs new file mode 100644 index 000000000..a86b9aab3 --- /dev/null +++ b/fuzz/fuzz_targets/generated.rs @@ -0,0 +1,163 @@ +//! Grammar-directed scripts through the pipeline, checked against the walker. +//! +//! `roundtrip` hands libfuzzer's bytes straight to the parser, which for a +//! language with real syntax means most of them are rejected before anything +//! interesting runs. This spends the same bytes on *grammar decisions* instead, +//! so every input is a valid script and coverage feedback is steering the shape +//! of the program rather than the spelling of it. +//! +//! `tests/fuzz.rs` runs the same generator over a few thousand seeded scripts +//! on every `cargo test`. This is the one that runs for hours. +//! +//! `cargo fuzz run generated` + +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rhai::grain::{Compiler, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +// The generator lives with the tests because that is the only thing that needs +// it, and putting it in the library would make it public API. This crate is +// outside the workspace and cannot depend on a test target, so it takes the +// source directly — inside a module, because the file opens with `//!`. +mod generate { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../tests/grain/corpus/generate.rs" + )); +} + +use generate::Generator; + +/// Both sides budgeted identically, or a script that runs out of operations on +/// one and not the other reads as a divergence. +fn engine() -> Engine { + let mut engine = Engine::new(); + engine.set_max_operations(200_000); + engine.set_max_string_size(8192); + engine.set_max_array_size(2048); + engine.set_max_map_size(64); + // Pinned for the reason `tests/fuzz.rs` pins them: rhai's defaults are + // `debug_assertions`-gated (`api/limits.rs:10-36`), so an unpinned harness + // explores one script space here and a different one under `cargo test`. + // Same numbers in both, so a finding from one reproduces in the other. + engine.set_max_expr_depths(64, 64); + engine.set_max_call_levels(64); + engine +} + +/// Run the walker, surviving a panic of its own making. +/// +/// Rhai's optimizer can leave a local's parse-time scope index stale, and +/// evaluating it then underflows (`eval/expr.rs:131`). It is reachable from an +/// ordinary script with none of this involved — `tests/fuzz.rs` pins a +/// reproducer — and a fuzzing run trips it often enough to end the run in +/// minutes, which costs far more than it finds. There is nothing to compare +/// against a side that did not finish, so those inputs are dropped. +/// +/// The hook is swapped for the duration because libfuzzer installs one that +/// aborts, which would defeat `catch_unwind`. It is swapped back immediately, +/// so a panic anywhere else in this target still ends the run as it should. +fn walk(engine: &Engine, ast: &rhai::AST, scope: &mut Scope) -> Option { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + engine.eval_ast_with_scope::(scope, ast) + })); + std::panic::set_hook(hook); + result.ok() +} + +type RhaiOutcome = Result>; + +/// A run reduced to something two of them can be compared on, scope included: +/// a VM that produced the right value while leaving the scope a different shape +/// has still got it wrong, and that is how the slot model fails. +fn outcome(scope: &Scope, result: RhaiOutcome) -> Option { + let scope: Vec = scope + .iter_raw() + .map(|(name, _, value)| format!("{name}={value:?}")) + .collect(); + + match result { + Ok(value) => Some(format!("ok {value:?} [{}]", scope.join(","))), + // The limits neither side counts towards in lockstep. `tests/fuzz.rs` + // documents why and asserts they stay rare; here they are simply + // dropped, because there is no way to assert a rate over one input. + Err(err) + if matches!( + *err, + rhai::EvalAltResult::ErrorTooManyOperations(..) + | rhai::EvalAltResult::ErrorStackOverflow(..) + | rhai::EvalAltResult::ErrorTooManyVariables(..) + ) => + { + None + } + Err(err) => Some(format!("err {err:?} [{}]", scope.join(","))), + } +} + +/// Run a script both ways under one engine. `None` if there is nothing to +/// compare — it did not parse, the walker did not survive it, or a limit +/// decided it. +fn compare(engine: &Engine, source: &str) -> Option<(String, String)> { + let ast = engine.compile(source).ok()?; + + let mut walker_scope = Scope::new(); + let walked = walk(engine, &ast, &mut walker_scope)?; + let expected = outcome(&walker_scope, walked)?; + + let program = Compiler::new().compile(&ast); + let mut vm_scope = Scope::new(); + let ours = if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(engine).eval_with_callbacks(&mut vm_scope, &program) + } else { + Vm::new(engine).eval_with_scope(&mut vm_scope, &program) + }; + Some((outcome(&vm_scope, ours)?, expected)) +} + +/// Whether a divergence is rhai's optimizer losing a local rather than a bug of +/// ours — see `rhai_drops_a_local_its_optimizer_still_refers_to` in +/// `tests/fuzz.rs`. +/// +/// The optimizer can delete a `let` whose variable is still read, leaving the +/// read pointing at whatever now sits at that scope index. Rhai answers with +/// another variable's value; we resolve by name and report it missing. There is +/// no agreeing with an AST that refers to a local it does not declare. +/// +/// Turning the optimizer off is what tells the two apart, and it runs only on a +/// divergence that already looks like this one — a fuzzer that quietly stopped +/// comparing would be worse than one that stops. +fn optimizer_lost_a_local(source: &str, ours: &str) -> bool { + if !ours.contains("ErrorVariableNotFound") { + return false; + } + let mut plain = engine(); + plain.set_optimization_level(rhai::OptimizationLevel::None); + compare(&plain, source).is_some_and(|(ours, expected)| ours == expected) +} + +fuzz_target!(|data: &[u8]| { + // Too few bytes to steer with, and the PRNG fallback would make every such + // input the same script. + if data.len() < 8 { + return; + } + + let source = Generator::from_bytes(data).script(); + let Some((ours, expected)) = compare(&engine(), &source) else { + return; + }; + if ours == expected || optimizer_lost_a_local(&source, &ours) { + return; + } + + assert_eq!( + ours, expected, + "the VM disagrees with the walker on:\n{source}" + ); +}); diff --git a/fuzz/fuzz_targets/load.rs b/fuzz/fuzz_targets/load.rs new file mode 100644 index 000000000..75d686f8d --- /dev/null +++ b/fuzz/fuzz_targets/load.rs @@ -0,0 +1,46 @@ +//! Arbitrary bytes into `Program::read`, then run whatever survives. +//! +//! This is the whole untrusted surface: a device is handed an artifact over a +//! link and executes it in place. The claim is total — any byte string either +//! fails to load, or loads into a chunk that runs without panicking, reading +//! outside itself, or running away. +//! +//! Running the survivors is the point rather than a bonus. A loader that +//! accepts a chunk it should not have has done nothing observable until +//! something executes it, so a target that only called `read` would miss the +//! failures that matter most. +//! +//! `cargo fuzz run load` + +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rhai::grain::{Program, Vm}; +use rhai::{Engine, Scope}; + +fuzz_target!(|data: &[u8]| { + let Ok(program) = Program::read(data) else { + return; + }; + + // `read` verifies before returning, and the VM's missing bounds checks + // rest on that. If it ever hands back something unverified, the contract + // is broken whether or not running it happens to work. + assert!( + program.verify().is_ok(), + "read returned a chunk that does not verify", + ); + + // Verification proves structure, not termination: a jump target that is + // in range is a valid infinite loop. That is not a gap a loader can close + // — no one can decide halting — which is why `Op::Tick` sits on every back + // edge and why a host running untrusted bytecode must set this. + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + engine.set_max_string_size(4096); + engine.set_max_array_size(1024); + engine.set_max_map_size(64); + engine.set_max_call_levels(16); + + let _ = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program); +}); diff --git a/fuzz/fuzz_targets/roundtrip.rs b/fuzz/fuzz_targets/roundtrip.rs new file mode 100644 index 000000000..59400d99d --- /dev/null +++ b/fuzz/fuzz_targets/roundtrip.rs @@ -0,0 +1,155 @@ +//! Arbitrary *scripts* through the whole pipeline, checked against the walker. +//! +//! The other target proves a hostile artifact cannot misbehave. This one +//! proves a well-formed one still means what it said: compile, write, read +//! back, run, and get what rhai's own evaluator got — the differential corpus +//! generalised from cases someone thought of to inputs nobody did. +//! +//! Both halves are load-bearing and they fail differently. A divergence here +//! is a compiler or VM bug; a panic there is a safety bug. +//! +//! `cargo fuzz run roundtrip` + +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rhai::grain::{Compiler, Program, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +/// Both sides must be budgeted identically, or a script that runs out of +/// operations on one and not the other reads as a divergence. +fn engine() -> Engine { + let mut engine = Engine::new(); + engine.set_max_operations(50_000); + engine.set_max_string_size(4096); + engine.set_max_array_size(1024); + engine.set_max_map_size(64); + engine.set_max_call_levels(16); + // Parse-time depth is `debug_assertions`-gated in rhai + // (`api/limits.rs:10-36`), so without this the set of scripts this target + // accepts depends on how it was built. Call depth above is already pinned + // for the same reason. + engine.set_max_expr_depths(64, 64); + engine +} + +/// A value, with the one difference between the two sides that is intended. +/// +/// Rhai renders a closure pointer `Fn*+("anon$..")` — a script function with a +/// captured environment attached — and ours `Fn("anon$..")`, because ours is +/// name-only and resolved at call time. That is the whole point: a `Script` +/// pointer carries an AST, and an AST is what an artifact must not contain. +/// The difference is deliberate, pinned by `a_closure_pointer_is_late_bound` in +/// `tests/scope.rs`, and left in the rendering rather than papered over there. +/// +/// Here it has to be papered over, or the first script whose value is a bare +/// closure ends the run — which is within a few thousand executions. +fn rendered(value: &Dynamic) -> String { + format!("{value:?}") + .replace("Fn*+(", "Fn(") + .replace("Fn*(", "Fn(") +} + +/// A run reduced to something two of them can be compared on, scope included: +/// a VM that produced the right value while leaving the scope a different +/// shape has still got it wrong, and that is how the slot model fails. +fn outcome(run: impl FnOnce(&mut Scope) -> Result>) -> String { + let mut scope = Scope::new(); + let result = run(&mut scope); + let scope: Vec = scope + .iter_raw() + .map(|(name, _, value)| rendered(value)) + .collect(); + + match result { + Ok(value) => format!("ok {} [{}]", rendered(&value), scope.join(",")), + // The limits are the one thing allowed to differ: the walker ticks per + // AST node and the VM per loop back-edge, and a callback crossing costs + // the two of them different numbers of call levels. Neither count will + // ever match, so a script that runs into one is dropped. + Err(err) + if matches!( + *err, + rhai::EvalAltResult::ErrorTooManyOperations(..) + | rhai::EvalAltResult::ErrorStackOverflow(..) + | rhai::EvalAltResult::ErrorTooManyVariables(..) + ) => + { + "budget".to_string() + } + Err(err) => format!("err {err:?} [{}]", scope.join(",")), + } +} + +/// Whether a divergence is rhai's optimizer losing a local rather than a bug of +/// ours — see `rhai_drops_a_local_its_optimizer_still_refers_to` in +/// `tests/fuzz.rs`. +/// +/// The optimizer can delete a `let` whose variable is still read, leaving the +/// read pointing at whatever now sits at that scope index. Rhai answers with +/// another variable's value; we resolve by name and report it missing. There is +/// no agreeing with an AST that refers to a local it does not declare. +/// +/// Turning the optimizer off is what tells the two apart, and it runs only on a +/// divergence that already looks like this one — a fuzzer that quietly stopped +/// comparing would be worse than one that stops. +fn optimizer_lost_a_local(source: &str, direct: &str) -> bool { + if !direct.contains("ErrorVariableNotFound") { + return false; + } + let mut plain = engine(); + plain.set_optimization_level(rhai::OptimizationLevel::None); + let Ok(ast) = plain.compile(source) else { + return false; + }; + let program = Compiler::new().compile(&ast); + + let expected = outcome(|scope| plain.eval_ast_with_scope::(scope, &ast)); + let ours = outcome(|scope| Vm::new(&plain).eval_with_scope(scope, &program)); + ours == expected +} + +fuzz_target!(|source: String| { + let engine = engine(); + let Ok(ast) = engine.compile(&source) else { + return; + }; + + let program = Compiler::new().compile(&ast); + + let expected = outcome(|scope| engine.eval_ast_with_scope::(scope, &ast)); + // A program that can hand a pointer to a native has to be run the way such + // a program is meant to be run, or every one of them reads as a divergence. + let shared = program + .makes_fn_pointers() + .then(|| Compiler::new().compile(&ast).into_shared()); + let run = |scope: &mut Scope| match &shared { + Some(shared) => Vm::new(&engine).eval_with_callbacks(scope, shared), + None => Vm::new(&engine).eval_with_scope(scope, &program), + }; + + let direct = outcome(run); + if expected == "budget" || direct == "budget" { + return; + } + if direct != expected && optimizer_lost_a_local(&source, &direct) { + return; + } + assert_eq!( + direct, expected, + "the VM disagrees with the walker on:\n{source}" + ); + + // A program that still fragments cannot be written, which is not a bug — + // it is the escape hatch doing its job. + let Ok(bytes) = program.write() else { + return; + }; + let reloaded = Program::read(&bytes).expect("what we wrote must read back"); + let loaded = outcome(|scope| Vm::new(&engine).eval_with_scope(scope, &reloaded)); + + assert_eq!( + loaded, expected, + "the artifact disagrees with the walker on:\n{source}" + ); +}); diff --git a/src/eval/chaining.rs b/src/eval/chaining.rs index 7c19eaf73..be16346ab 100644 --- a/src/eval/chaining.rs +++ b/src/eval/chaining.rs @@ -6,8 +6,8 @@ use crate::ast::{ASTFlags, BinaryExpr, Expr, OpAssignment}; use crate::engine::{FN_IDX_GET, FN_IDX_SET}; use crate::types::dynamic::Union; use crate::{ - calc_fn_hash, Dynamic, Engine, ExclusiveRange, FnArgsVec, InclusiveRange, OnceCell, Position, - RhaiResult, RhaiResultOf, Scope, ERR, + calc_fn_hash, Dynamic, Engine, FnArgsVec, OnceCell, Position, RhaiResult, RhaiResultOf, Scope, + ERR, }; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -108,7 +108,7 @@ impl Engine { /// Panics if the target object is shared. /// /// Shared objects should be handled (dereferenced) before calling this method. - fn get_indexed_mut<'t>( + pub(crate) fn get_indexed_mut<'t>( &self, global: &mut GlobalRuntimeState, caches: &mut Caches, @@ -354,8 +354,8 @@ impl Engine { // Range index on empty string - empty slice Err(typ) - if (typ == std::any::type_name::() - || typ == std::any::type_name::()) + if (typ == std::any::type_name::() + || typ == std::any::type_name::()) && s.is_empty() => { let value = s.clone().into(); @@ -369,9 +369,9 @@ impl Engine { } // Range index - slice - Err(typ) if typ == std::any::type_name::() => { + Err(typ) if typ == std::any::type_name::() => { // val_str[range] - let range = idx.read_lock::().unwrap().clone(); + let range = idx.read_lock::().unwrap().clone(); let chars_count = s.chars().count(); let start = if range.start >= 0 { @@ -405,9 +405,9 @@ impl Engine { exclusive: true, }) } - Err(typ) if typ == std::any::type_name::() => { + Err(typ) if typ == std::any::type_name::() => { // val_str[range] - let range = idx.read_lock::().unwrap().clone(); + let range = idx.read_lock::().unwrap().clone(); let chars_count = s.chars().count(); let start = if *range.start() >= 0 { diff --git a/src/eval/eval_context.rs b/src/eval/eval_context.rs index ee7e86d1f..fc9eadf4f 100644 --- a/src/eval/eval_context.rs +++ b/src/eval/eval_context.rs @@ -489,6 +489,14 @@ fn _call_fn_raw( let args_len = args.len(); if native_only { + // The functions rhai answers by name are all reserved names, and a + // reserved name is what makes a call native-only — so without this the + // branch routes straight past their only implementation. See + // `Engine::exec_syntactic_fn_call`. + if let Some(result) = engine.exec_syntactic_fn_call(fn_name, args, Position::NONE) { + return result; + } + let hash = calc_fn_hash(None, fn_name, args_len); return engine diff --git a/src/eval/expr.rs b/src/eval/expr.rs index 06227fecb..168dc84a4 100644 --- a/src/eval/expr.rs +++ b/src/eval/expr.rs @@ -55,6 +55,54 @@ impl Engine { ) -> RhaiResultOf> { // Make sure that the pointer indirection is taken only when absolutely necessary. + // A bare script-function name is a function pointer, not a variable read. + // + // Checked ahead of `always_search_scope` rather than inside the match: + // that flag means "do not trust the parse-time variable indices", and a + // name resolving to a function has no index to distrust. Gating this on + // it made the name stop resolving for the rest of a run as soon as + // anything set it — an `eval` that changes the scope, a variable + // resolver that does, or a program still holding an AST fragment — so + // `fn f(x) { x } eval("let m = 1;"); [1].map(f)` reported `f` as an + // unknown variable while the same script without the `eval` worked. + // + // Still only for a variable with no cached index, which is what it was + // before: an index means the parser resolved the name to a real + // variable, and that variable keeps winning over a function sharing its + // name. + #[cfg(not(feature = "no_function"))] + if let Expr::Variable(v, None, ..) = expr { + if let Some(func) = global + .lib + .iter() + .flat_map(|m| m.iter_fn()) + .filter(|(f, _)| f.is_script()) + .filter(|(_, m)| m.name == v.1.as_str()) + .map(|(f, _)| f) + .next() + { + // Embedded environment for scripted function + let env = func + .get_shared_encapsulated_environ() + .cloned() + .unwrap_or_else(|| { + // Create a new environment with the current module + crate::Shared::new((&*global).into()) + }); + + let val: Dynamic = crate::FnPtr { + name: v.1.clone(), + curry: <_>::default(), + env: Some(env), + typ: crate::types::fn_ptr::FnPtrType::Script( + func.get_script_fn_def().cloned().unwrap(), + ), + } + .into(); + return Ok(val.into()); + } + } + let index = match expr { // Check if the variable is `this` Expr::ThisPtr(..) => unreachable!("Expr::ThisPtr should have been handled outside"), @@ -66,38 +114,6 @@ impl Engine { #[cfg(not(feature = "no_module"))] debug_assert!(v.2.is_empty(), "variable should not be namespace-qualified"); - // Scripted function with the same name - #[cfg(not(feature = "no_function"))] - if let Some(func) = global - .lib - .iter() - .flat_map(|m| m.iter_fn()) - .filter(|(f, _)| f.is_script()) - .filter(|(_, m)| m.name == v.1.as_str()) - .map(|(f, _)| f) - .next() - { - // Embedded environment for scripted function - let env = func - .get_shared_encapsulated_environ() - .cloned() - .unwrap_or_else(|| { - // Create a new environment with the current module - crate::Shared::new((&*global).into()) - }); - - let val: Dynamic = crate::FnPtr { - name: v.1.clone(), - curry: <_>::default(), - env: Some(env), - typ: crate::types::fn_ptr::FnPtrType::Script( - func.get_script_fn_def().cloned().unwrap(), - ), - } - .into(); - return Ok(val.into()); - } - v.0.map_or(0, NonZeroUsize::get) } diff --git a/src/func/call.rs b/src/func/call.rs index 9e1ac16d7..013607bef 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -546,6 +546,57 @@ impl Engine { } } + /// The functions answered by name rather than by dispatch. + /// + /// `type_of` and `is_shared` have no registered implementation anywhere — + /// this *is* their implementation, and the rest of the names here exist only + /// as syntax, so reaching one of them by dispatch is always an error. + /// + /// `None` means the name is not one of these and should be dispatched + /// normally. `Some` is the whole answer: the value, or the error for a + /// spelling that has none — `type_of` with the wrong number of arguments, or + /// a name that is only ever syntax. + /// + /// Shared with [`NativeCallContext::call_fn_raw`], which reaches names it + /// decided were native-only without going through [`Self::exec_fn_call`] at + /// all. Keeping one copy is the point: these used to be answerable only + /// through the script path, so a host calling `call_fn_raw("type_of", ..)` + /// — and the bytecode VM, which dispatches every call that way — got + /// `ErrorFunctionNotFound` for a function rhai does implement. + pub(crate) fn exec_syntactic_fn_call( + &self, + fn_name: &str, + args: &FnCallArgs, + pos: Position, + ) -> Option { + let only_syntax = match fn_name { + // Handle type_of() + KEYWORD_TYPE_OF if args.len() == 1 => { + let typ = self.get_interned_string(self.map_type_name(args[0].type_name())); + return Some(Ok(typ.into())); + } + + #[cfg(not(feature = "no_closure"))] + crate::engine::KEYWORD_IS_SHARED if args.len() == 1 => { + return Some(Ok(args[0].is_shared().into())) + } + #[cfg(not(feature = "no_closure"))] + crate::engine::KEYWORD_IS_SHARED => true, + + #[cfg(not(feature = "no_function"))] + crate::engine::KEYWORD_IS_DEF_FN => true, + + KEYWORD_TYPE_OF | KEYWORD_FN_PTR | KEYWORD_EVAL | KEYWORD_IS_DEF_VAR + | KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY => true, + + _ => false, + }; + + only_syntax.then(|| { + Err(ERR::ErrorFunctionNotFound(self.gen_fn_call_signature(fn_name, args), pos).into()) + }) + } + /// # Main Entry-Point (By Name) /// /// Perform an actual function call, native Rust or scripted, by name, taking care of special functions. @@ -570,32 +621,10 @@ impl Engine { pos: Position, ) -> RhaiResultOf<(Dynamic, bool)> { // These may be redirected from method style calls. - if hashes.is_native_only() - && match fn_name { - // Handle type_of() - KEYWORD_TYPE_OF if args.len() == 1 => { - let typ = self.get_interned_string(self.map_type_name(args[0].type_name())); - return Ok((typ.into(), false)); - } - - #[cfg(not(feature = "no_closure"))] - crate::engine::KEYWORD_IS_SHARED if args.len() == 1 => { - return Ok((args[0].is_shared().into(), false)) - } - #[cfg(not(feature = "no_closure"))] - crate::engine::KEYWORD_IS_SHARED => true, - - #[cfg(not(feature = "no_function"))] - crate::engine::KEYWORD_IS_DEF_FN => true, - - KEYWORD_TYPE_OF | KEYWORD_FN_PTR | KEYWORD_EVAL | KEYWORD_IS_DEF_VAR - | KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY => true, - - _ => false, + if hashes.is_native_only() { + if let Some(result) = self.exec_syntactic_fn_call(fn_name, args, pos) { + return result.map(|value| (value, false)); } - { - let sig = self.gen_fn_call_signature(fn_name, args); - return Err(ERR::ErrorFunctionNotFound(sig, pos).into()); } // Check for data race. diff --git a/src/func/native.rs b/src/func/native.rs index 99e670b58..13511227e 100644 --- a/src/func/native.rs +++ b/src/func/native.rs @@ -523,6 +523,18 @@ impl<'a> NativeCallContext<'a> { let args_len = args.len(); if native_only { + // A reserved name is native-only *because* it is reserved, and the + // ones rhai answers by name rather than by dispatch are all + // reserved — so this branch would otherwise route straight past + // their only implementation. `type_of` has none to find afterwards, + // and reported itself as an unknown function. + if let Some(result) = + self.engine() + .exec_syntactic_fn_call(fn_name, args, self.call_position()) + { + return result; + } + return self .engine() .exec_native_fn_call( diff --git a/src/grain/bytecode/chain.rs b/src/grain/bytecode/chain.rs new file mode 100644 index 000000000..ddc3c494f --- /dev/null +++ b/src/grain/bytecode/chain.rs @@ -0,0 +1,229 @@ +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +/// One step along `a.b[i].c(x)`. +/// +/// Steps live in the program's chain pool rather than in the instruction +/// stream, because a chain is walked by one instruction rather than several. +/// It has to be: the walk holds a `&mut` into the container at every level, and +/// a borrow cannot survive a trip round the dispatch loop. That is also what +/// makes it correct — rhai holds the same references, so a mutation partway +/// down a chain lands in the same place rather than in a copy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Step { + /// `[i]`, where the index was evaluated onto the operand stack before the + /// chain instruction ran, at `operand` from the first of them. + /// + /// Pre-evaluating mirrors rhai, which collects every index in a chain into + /// `idx_values` before walking it (`eval/chaining.rs:568`). It has to + /// happen first: evaluating an index halfway down would need the operand + /// stack while a borrow of the container is live. + /// + /// Rhai reports `a[10]` out of bounds against the `10` rather than against + /// the chain (`eval/chaining.rs:694`). + /// + /// `bracket` is the other position rhai keeps for a step, and the two are + /// not interchangeable: `pos` is where the index expression starts and + /// `bracket` is the `[` in front of it (`op_pos`, `eval/chaining.rs:695`). + /// An out-of-bounds index is blamed on the first and indexing something + /// that cannot be indexed on the second, so `a[0][5]` where `a[0]` is not + /// indexable names the *second* `[` — the step that failed — and a chain + /// carrying one position between it and its neighbours would name the + /// wrong one. + Index { + /// Where the index sits on the operand stack. + operand: u16, + /// Where the index expression starts. + pos: rhai::Position, + /// The `[` in front of it. + bracket: rhai::Position, + }, + + /// `.name`, which is a key lookup on a map and a getter call on anything + /// else — the distinction rhai makes at runtime, not at parse time + /// (`eval/chaining.rs:898`). + Property { + /// The bare name, for a map key and for error messages. + name: u32, + /// `get$name`. + getter: u32, + /// `set$name`, for the write-back. + setter: u32, + /// Where the property is in the source. + pos: rhai::Position, + }, + + /// `.name(args)`, with the receiver as the first argument by reference. + Method { + /// The name of the method + name: u32, + /// How many arguments, not counting the receiver. + argc: u8, + /// Where the first of them sits on the operand stack. + operand: u16, + /// Where the call is in the source. + pos: rhai::Position, + }, +} + +impl Step { + /// Where this step is in the source. + /// + /// Every step carries one, and it is the one place diagnostics are not + /// strippable — four bytes per step, in the chain pool rather than the + /// position table. That is not an oversight twice over: a chain is a single + /// instruction, so the one entry the table holds for it cannot say which of + /// `a.b[i].c()` failed, and rhai blames the step rather than the chain for + /// all three kinds. An index is reported against its index expression, a + /// property against the property (`eval/chaining.rs:1039`), a method + /// against the call (`:904`). + #[must_use] + pub fn pos(&self) -> rhai::Position { + match self { + Step::Index { pos, .. } | Step::Property { pos, .. } | Step::Method { pos, .. } => *pos, + } + } +} + +/// What a chain does when it gets to the end. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Tail { + /// Push the value the chain arrived at. + Read, + /// Assign the top of the operand stack to it, optionally through an + /// operator, and push unit. + Assign { + /// Index into the op-assignment pool; absent for a plain `=`. + op: Option, + }, +} + +/// Where a chain starts. +/// +/// The distinction is whether the root has an identity to write back into. +/// Rhai draws the same line and in the same place: a variable root becomes a +/// `Target` into the scope entry, and anything else is evaluated into a +/// temporary and walked there (`eval/chaining.rs:547-571`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Root { + /// A local, by slot. + /// + /// The only root a chain can write *through*: a mutation partway down has + /// to land in the scope entry, and walking a copy of it would lose the + /// mutation. + Local { + /// The slot index + slot: u16, + /// Names it in `ErrorAssignmentToConstant`. + name: u32, + }, + + /// A variable no slot addresses: one the caller put in the `Scope`, one a + /// resolver answers for, or a module's constant. + /// + /// Whether it can be written through is not known until it is looked up, + /// which is the whole difference from [`Root::Local`]. Rhai decides the + /// same way and at the same moment: `search_namespace` hands back a + /// `Target`, and a scope entry becomes a reference where a resolver's + /// answer or a module's constant becomes a read-only temporary + /// (`eval/expr.rs:120-155`). + /// + /// Carries its own position because the lookup can fail and + /// `ErrorVariableNotFound` is reported against the variable, not the + /// chain. That costs nothing extra: chain positions already live in this + /// pool rather than in the strippable table, for the reason [`Step::pos`] + /// gives. + Named { + /// The name of the variable + name: u32, + /// Where the variable is in the source. + pos: rhai::Position, + }, + + /// The frame's receiver. + /// + /// Grouped with the two above rather than with [`Root::Temporary`], and the + /// distinction is the whole reason this variant exists: `this.push(1)` has + /// to mutate the caller's value, and a temporary would walk a copy and drop + /// the mutation silently. + /// + /// Carries its own position because the chain instruction's table entry is + /// the `.` or the `[`, while `ErrorUnboundThis` is reported against the + /// `this` (`eval/chaining.rs:519-527`) — two positions one instruction + /// cannot give. [`Root::Named`] carries one for the same reason. + This { + /// Where the `this` is in the source. + pos: rhai::Position, + }, + + /// A value the instruction takes off the operand stack, pushed above the + /// step operands. + /// + /// `[1, 2, 3].len()`, `f().x`, `(a + b).to_string()`. Nothing is written + /// back, because there is nowhere to write it back to — and nothing can be + /// assigned to one, because rhai's parser refuses `f().x = 1` before this + /// ever sees it (`eval/chaining.rs:559`). + Temporary, +} + +/// A whole `a.b[i].c` chain. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Chain { + /// Where the chain starts. + pub root: Root, + /// The steps, in source order. + pub steps: Vec, + /// What happens at the end. + pub tail: Tail, + /// How many operand-stack values the *steps* consume, so the VM knows + /// where they start. Not the whole instruction's appetite — see + /// [`Chain::consumes`]. + pub operands: u16, +} + +impl Chain { + /// Whether the chain takes a value off the operand stack to store. + #[must_use] + pub fn assigns(&self) -> bool { + matches!(self.tail, Tail::Assign { .. }) + } + + /// Whether the root itself arrives on the operand stack. + /// + /// Only a temporary does. The other two are reached where they live — by + /// slot or by name — and getting this wrong is not a small mistake: + /// [`Chain::consumes`] is what the verifier models a chunk's whole operand + /// depth on, and what the VM finds its operands with. + #[must_use] + pub fn roots_on_stack(&self) -> bool { + match self.root { + Root::Local { .. } | Root::Named { .. } | Root::This { .. } => false, + Root::Temporary => true, + } + } + + /// Everything the instruction takes off the operand stack. + /// + /// Pushed in that order — step operands, then the root, then the value + /// being assigned — which is rhai's evaluation order and not the reading + /// order: it collects a chain's indices and arguments *before* it evaluates + /// what they are being applied to (`eval/chaining.rs:498-524` then `:562`). + #[must_use] + pub fn consumes(&self) -> usize { + self.operands as usize + usize::from(self.roots_on_stack()) + usize::from(self.assigns()) + } + + /// Whether walking this chain can change what it walks over. + /// + /// A read-only chain needs no write-back at all, which is worth knowing: + /// write-back on a temporary calls a setter, and calling one where rhai + /// would not is an observable difference on a host type. + #[must_use] + pub fn mutates(&self) -> bool { + matches!(self.tail, Tail::Assign { .. }) + || self + .steps + .iter() + .any(|step| matches!(step, Step::Method { .. })) + } +} diff --git a/src/grain/bytecode/chunk.rs b/src/grain/bytecode/chunk.rs new file mode 100644 index 000000000..b5bf4f82a --- /dev/null +++ b/src/grain/bytecode/chunk.rs @@ -0,0 +1,69 @@ +use crate::grain::bytecode::code::disassemble; +use crate::grain::bytecode::Op; + +/// One body of code: the top-level program, or one script function. +/// +/// Metadata only. Every chunk in a program shares a single instruction buffer, +/// concatenated in order, and a chunk names its span of it. That keeps one +/// position table and one instruction address across the whole program, so a +/// device reporting where it failed reports one number. +/// +/// `max_stack` exists so the VM can size its operand stack rather than growing +/// it. The compiler emits an upper bound it can compute without a depth walk — +/// one per instruction — and then replaces it with the high water the verifier +/// actually measured. On a device that difference is the whole reservation: a +/// chunk of 25 instructions rarely stacks more than three values. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Chunk { + entry: u32, + end: u32, + max_stack: u16, +} + +impl Chunk { + pub(crate) fn new(entry: u32, end: u32, max_stack: u16) -> Self { + Self { + entry, + end, + max_stack, + } + } + + /// Where execution starts, as an offset into the program's code. + #[must_use] + pub fn entry(&self) -> u32 { + self.entry + } + + /// One past the last byte of this chunk. + #[must_use] + pub fn end(&self) -> u32 { + self.end + } + + /// The deepest the operand stack gets, as proven by the verifier. + #[must_use] + pub fn max_stack(&self) -> u16 { + self.max_stack + } + + pub(crate) fn set_max_stack(&mut self, max_stack: u16) { + self.max_stack = max_stack; + } + + /// This chunk's slice of a program's code. + #[must_use] + pub fn body<'c>(&self, code: &'c [u8]) -> &'c [u8] { + code.get(self.entry as usize..self.end as usize) + .unwrap_or_default() + } + + /// The instructions, paired with their addresses in the program. + /// + /// For reading, not for running — reconstructing an [`Op`] is exactly the + /// work the byte encoding exists to avoid. + pub fn ops<'c>(&self, code: &'c [u8]) -> impl Iterator + 'c { + let entry = self.entry as usize; + disassemble(self.body(code)).map(move |(at, op)| (at + entry, op)) + } +} diff --git a/src/grain/bytecode/code.rs b/src/grain/bytecode/code.rs new file mode 100644 index 000000000..d6ede071f --- /dev/null +++ b/src/grain/bytecode/code.rs @@ -0,0 +1,1155 @@ +//! The executable form: instructions as bytes, run in place. +//! +//! [`Op`] is what the compiler emits and what a disassembly shows. It is not +//! what runs. A `Vec` costs sixteen bytes an instruction and has to be +//! built at load, which is most of what this project exists to avoid — so a +//! program's code is a byte slice, and a loaded program borrows it from the +//! artifact rather than decoding it into anything. +//! +//! ## Why fixed fields rather than varints +//! +//! Everything else in an artifact is LEB128, because everything else is read +//! once. Instructions are read on every execution, so their operands are +//! fixed-width little-endian at known offsets: decoding is a match on the tag +//! and a couple of loads, with no loop per field. That costs about one byte per +//! operand against the varint form and buys a dispatch loop that does not +//! decode. +//! +//! ## Jumps are byte offsets +//! +//! Instructions vary in length, so a jump names a byte offset rather than an +//! instruction index. [`assemble`] resolves the compiler's indices into offsets +//! once, and [`verify`](super::verify) proves every one of them lands on an +//! instruction boundary — without which a jump into the middle of an operand +//! would decode whatever the operand's bytes happen to look like. + +use alloc::borrow::Cow; + +use crate::grain::bytecode::{Op, Receiver}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +/// Instruction tags. +/// +/// Written out rather than derived from [`Op`]'s order, so reordering the enum +/// for readability cannot change what a program means. Append only. +/// +/// Operators get their own tag rather than an optional field, so the common +/// call pays nothing for the one that carries a token. +pub mod tag { + /// [`Op::Const`](super::Op::Const). + pub const CONST: u8 = 0x01; + /// [`Op::Unit`](super::Op::Unit). + pub const UNIT: u8 = 0x02; + /// [`Op::Bool`](super::Op::Bool) holding `false`. + pub const FALSE: u8 = 0x03; + /// [`Op::Bool`](super::Op::Bool) holding `true`. + pub const TRUE: u8 = 0x04; + /// [`Op::LoadLocal`](super::Op::LoadLocal). + pub const LOAD_LOCAL: u8 = 0x05; + /// [`Op::StoreLocal`](super::Op::StoreLocal). + pub const STORE_LOCAL: u8 = 0x06; + /// [`Op::AssignLocal`](super::Op::AssignLocal) with a plain `=`. + pub const ASSIGN_LOCAL: u8 = 0x07; + /// [`Op::AssignLocal`](super::Op::AssignLocal) through an operator. + pub const ASSIGN_LOCAL_OP: u8 = 0x08; + /// [`Op::DeclareLocal`](super::Op::DeclareLocal) for a `let`. + pub const DECLARE_LOCAL: u8 = 0x09; + /// [`Op::DeclareLocal`](super::Op::DeclareLocal) for a `const`. + pub const DECLARE_CONST: u8 = 0x0a; + /// [`Op::Pop`](super::Op::Pop). + pub const POP: u8 = 0x0b; + /// [`Op::Jump`](super::Op::Jump). + pub const JUMP: u8 = 0x0c; + /// [`Op::JumpIfTrue`](super::Op::JumpIfTrue). + pub const JUMP_IF_TRUE: u8 = 0x0d; + /// [`Op::JumpIfFalse`](super::Op::JumpIfFalse). + pub const JUMP_IF_FALSE: u8 = 0x0e; + /// [`Op::Call`](super::Op::Call) to an ordinary function. + pub const CALL: u8 = 0x0f; + /// [`Op::Call`](super::Op::Call) to an operator. + pub const CALL_OP: u8 = 0x10; + /// [`Op::UnwindTo`](super::Op::UnwindTo). + pub const UNWIND_TO: u8 = 0x11; + /// [`Op::Tick`](super::Op::Tick). + pub const TICK: u8 = 0x12; + /// [`Op::Return`](super::Op::Return). + pub const RETURN: u8 = 0x13; + /// [`Op::EvalAst`](super::Op::EvalAst) that rewinds the scope. + pub const EVAL_AST: u8 = 0x14; + /// [`Op::EvalAst`](super::Op::EvalAst) that keeps what it declared. + pub const EVAL_AST_KEEP: u8 = 0x15; + /// [`Op::Chain`](super::Op::Chain). + pub const CHAIN: u8 = 0x16; + /// [`Op::MakeArray`](super::Op::MakeArray). + pub const MAKE_ARRAY: u8 = 0x17; + /// [`Op::Switch`](super::Op::Switch). + pub const SWITCH: u8 = 0x18; + /// [`Op::LoadNamed`](super::Op::LoadNamed). + pub const LOAD_NAMED: u8 = 0x19; + /// [`Op::AssignNamed`](super::Op::AssignNamed) with a plain `=`. + pub const ASSIGN_NAMED: u8 = 0x1a; + /// [`Op::AssignNamed`](super::Op::AssignNamed) through an operator. + pub const ASSIGN_NAMED_OP: u8 = 0x1b; + /// [`Op::Throw`](super::Op::Throw). + pub const THROW: u8 = 0x1c; + /// [`Op::IterInit`](super::Op::IterInit). + pub const ITER_INIT: u8 = 0x1d; + /// [`Op::IterNext`](super::Op::IterNext). + pub const ITER_NEXT: u8 = 0x1e; + /// [`Op::IterDrop`](super::Op::IterDrop). + pub const ITER_DROP: u8 = 0x1f; + /// [`Op::StoreShared`](super::Op::StoreShared). + pub const STORE_SHARED: u8 = 0x20; + /// [`Op::IterNext`](super::Op::IterNext) that also pushes the count. + pub const ITER_NEXT_INDEXED: u8 = 0x21; + /// [`Op::PopHandler`](super::Op::PopHandler). + pub const POP_HANDLER: u8 = 0x22; + /// [`Op::PushHandler`](super::Op::PushHandler) for a bare `catch`. + pub const PUSH_HANDLER: u8 = 0x23; + /// [`Op::PushHandler`](super::Op::PushHandler) binding a catch variable. + pub const PUSH_HANDLER_VAR: u8 = 0x24; + /// [`Op::InterpolateStart`](super::Op::InterpolateStart). + pub const INTERPOLATE_START: u8 = 0x25; + /// [`Op::InterpolateAppend`](super::Op::InterpolateAppend). + pub const INTERPOLATE_APPEND: u8 = 0x26; + /// [`Op::InterpolateEnd`](super::Op::InterpolateEnd). + pub const INTERPOLATE_END: u8 = 0x27; + /// [`Op::MakeFnPtr`](super::Op::MakeFnPtr). + pub const MAKE_FN_PTR: u8 = 0x28; + /// [`Op::Curry`](super::Op::Curry). + pub const CURRY: u8 = 0x29; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) in call position. + pub const CALL_FN_PTR: u8 = 0x2a; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) in method position. + pub const CALL_FN_PTR_METHOD: u8 = 0x2b; + /// [`Op::Share`](super::Op::Share). + pub const SHARE: u8 = 0x2c; + /// [`Op::ShareNamed`](super::Op::ShareNamed). + pub const SHARE_NAMED: u8 = 0x2d; + /// [`Op::LoadShared`](super::Op::LoadShared). + pub const LOAD_SHARED: u8 = 0x2e; + /// [`Op::MakeClosure`](super::Op::MakeClosure). + pub const MAKE_CLOSURE: u8 = 0x2f; + /// [`Op::IsShared`](super::Op::IsShared). + pub const IS_SHARED: u8 = 0x30; + /// [`Op::Checkpoint`](super::Op::Checkpoint). + pub const CHECKPOINT: u8 = 0x31; + /// [`Op::CheckSize`](super::Op::CheckSize) against the array limit. + pub const CHECK_ARRAY_SIZE: u8 = 0x32; + /// [`Op::CheckSize`](super::Op::CheckSize) against the map limit. + pub const CHECK_MAP_SIZE: u8 = 0x33; + /// [`Op::MakeMap`](super::Op::MakeMap). + pub const MAKE_MAP: u8 = 0x34; + /// [`Op::CallRef`](super::Op::CallRef) through [`Receiver::Local`](super::Receiver::Local). + pub const CALL_LOCAL_REF: u8 = 0x35; + /// [`Op::Rotate`](super::Op::Rotate). + pub const ROTATE: u8 = 0x36; + /// [`Op::CallRef`](super::Op::CallRef) through [`Receiver::Named`](super::Receiver::Named). + pub const CALL_NAMED_REF: u8 = 0x37; + /// [`Op::LoadSharedNamed`](super::Op::LoadSharedNamed). + pub const LOAD_SHARED_NAMED: u8 = 0x38; + /// [`Op::LoadThis`](super::Op::LoadThis). + pub const LOAD_THIS: u8 = 0x39; + /// [`Op::LoadThisShared`](super::Op::LoadThisShared). + pub const LOAD_THIS_SHARED: u8 = 0x3a; + /// [`Op::RequireThis`](super::Op::RequireThis). + pub const REQUIRE_THIS: u8 = 0x3b; + /// [`Op::AssignThis`](super::Op::AssignThis) with a plain `=`. + pub const ASSIGN_THIS: u8 = 0x3c; + /// [`Op::AssignThis`](super::Op::AssignThis) through an operator. + pub const ASSIGN_THIS_OP: u8 = 0x3d; + /// [`Op::CallRef`](super::Op::CallRef) through [`Receiver::This`](super::Receiver::This). + pub const CALL_THIS_REF: u8 = 0x3e; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) on a local, which is written back to. + pub const CALL_FN_PTR_ON_LOCAL: u8 = 0x3f; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) on a variable no slot names. + pub const CALL_FN_PTR_ON_NAMED: u8 = 0x40; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) on the frame's receiver. + pub const CALL_FN_PTR_ON_THIS: u8 = 0x41; +} + +/// How wide each tag's instruction is, with 0 for the tags that are not one. +/// +/// A table rather than a match because the dispatch loop needs the width of +/// every instruction it executes: matching on the tag twice, once to advance +/// and once to act, is a branch per instruction bought for nothing. +static WIDTHS: [u8; 256] = { + let mut widths = [0u8; 256]; + + widths[tag::UNIT as usize] = 1; + widths[tag::FALSE as usize] = 1; + widths[tag::TRUE as usize] = 1; + widths[tag::POP as usize] = 1; + widths[tag::TICK as usize] = 1; + widths[tag::CHECKPOINT as usize] = 1; + widths[tag::MAKE_MAP as usize] = 3; + widths[tag::CHECK_ARRAY_SIZE as usize] = 3; + widths[tag::CHECK_MAP_SIZE as usize] = 3; + widths[tag::RETURN as usize] = 1; + widths[tag::THROW as usize] = 1; + widths[tag::ITER_INIT as usize] = 1; + widths[tag::ITER_DROP as usize] = 1; + + widths[tag::STORE_SHARED as usize] = 3; + + widths[tag::ITER_NEXT as usize] = 5; + widths[tag::ITER_NEXT_INDEXED as usize] = 5; + widths[tag::POP_HANDLER as usize] = 1; + widths[tag::INTERPOLATE_START as usize] = 1; + widths[tag::INTERPOLATE_APPEND as usize] = 1; + widths[tag::INTERPOLATE_END as usize] = 1; + widths[tag::MAKE_FN_PTR as usize] = 1; + widths[tag::IS_SHARED as usize] = 1; + + widths[tag::CURRY as usize] = 2; + widths[tag::ROTATE as usize] = 2; + widths[tag::CALL_FN_PTR as usize] = 2; + widths[tag::CALL_FN_PTR_METHOD as usize] = 2; + + widths[tag::SHARE as usize] = 3; + widths[tag::SHARE_NAMED as usize] = 3; + widths[tag::LOAD_SHARED as usize] = 3; + widths[tag::LOAD_SHARED_NAMED as usize] = 3; + + // `this` is a register, so none of these needs an operand to address it. + widths[tag::LOAD_THIS as usize] = 1; + widths[tag::LOAD_THIS_SHARED as usize] = 1; + widths[tag::REQUIRE_THIS as usize] = 1; + widths[tag::ASSIGN_THIS as usize] = 1; + widths[tag::ASSIGN_THIS_OP as usize] = 3; + widths[tag::CALL_THIS_REF as usize] = 4; + + // The receiver's value is on the stack for all of these; only where it came + // from differs, and only two of them need an operand to say it. + widths[tag::CALL_FN_PTR_ON_LOCAL as usize] = 4; + widths[tag::CALL_FN_PTR_ON_NAMED as usize] = 4; + widths[tag::CALL_FN_PTR_ON_THIS as usize] = 2; + widths[tag::MAKE_CLOSURE as usize] = 3; + widths[tag::PUSH_HANDLER as usize] = 5; + widths[tag::PUSH_HANDLER_VAR as usize] = 7; + + widths[tag::CONST as usize] = 3; + widths[tag::LOAD_LOCAL as usize] = 3; + widths[tag::STORE_LOCAL as usize] = 3; + widths[tag::DECLARE_LOCAL as usize] = 3; + widths[tag::DECLARE_CONST as usize] = 3; + widths[tag::UNWIND_TO as usize] = 3; + widths[tag::EVAL_AST as usize] = 3; + widths[tag::EVAL_AST_KEEP as usize] = 3; + widths[tag::CHAIN as usize] = 3; + widths[tag::MAKE_ARRAY as usize] = 3; + widths[tag::SWITCH as usize] = 3; + widths[tag::LOAD_NAMED as usize] = 3; + widths[tag::ASSIGN_NAMED as usize] = 3; + + widths[tag::ASSIGN_NAMED_OP as usize] = 5; + + widths[tag::CALL as usize] = 4; + + widths[tag::ASSIGN_LOCAL as usize] = 5; + widths[tag::JUMP as usize] = 5; + widths[tag::JUMP_IF_TRUE as usize] = 5; + widths[tag::JUMP_IF_FALSE as usize] = 5; + + widths[tag::CALL_OP as usize] = 6; + widths[tag::CALL_LOCAL_REF as usize] = 6; + widths[tag::CALL_NAMED_REF as usize] = 6; + + widths[tag::ASSIGN_LOCAL_OP as usize] = 7; + + widths +}; + +/// How many bytes the instruction at `at` occupies, or `None` if the tag is +/// unknown or the operands run past the end. +#[must_use] +#[inline] +pub fn width(code: &[u8], at: usize) -> Option { + let size = WIDTHS[*code.get(at)? as usize] as usize; + if size == 0 { + return None; + } + // An instruction whose operands are cut off is not an instruction. + (at + size <= code.len()).then_some(size) +} + +/// Read a `u16` operand at `at`. +/// +/// Returns `None` past the end rather than panicking. The verifier makes that +/// unreachable for any program the VM will run, but the check is a load's worth +/// of cost and it means nothing has to be trusted. +#[must_use] +#[inline] +pub fn u16_at(code: &[u8], at: usize) -> Option { + Some(u16::from_le_bytes(code.get(at..at + 2)?.try_into().ok()?)) +} + +/// Read a `u32` operand at `at`. +#[must_use] +#[inline] +pub fn u32_at(code: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(code.get(at..at + 4)?.try_into().ok()?)) +} + +/// Why a lowering could not be turned into bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AssembleError { + /// A pool grew past what a `u16` operand can name. The compiler falls back + /// to a whole-program fragment rather than emitting a truncated index. + PoolTooLarge { + /// Which pool overflowed + what: &'static str, + /// How many entries it holds + entries: usize, + }, + /// A jump naming an instruction that does not exist. A compiler bug. + JumpOutOfRange { + /// Index of the jump instruction + at: usize, + /// The instruction index it names + target: u32, + }, + /// The same, for a jump that lives in a switch table rather than in the + /// code. + SwitchTargetOutOfRange { + /// Index into the switch pool + table: usize, + /// The instruction index the entry names + target: u32, + }, + /// A chunk longer than a `u32` of bytes. + ChunkTooLarge { + /// How long the chunk is + bytes: usize, + }, +} + +/// Pack instructions into their executable form. +/// +/// Two passes: the first measures each instruction so jump targets can be +/// turned from indices into byte offsets, the second writes. Also returns the +/// byte offset of each instruction, so the position table can be re-keyed from +/// indices onto addresses. +/// +/// # Errors +/// +/// [`AssembleError`] for anything that cannot be expressed in the operand +/// widths. All of them are compiler bugs except [`AssembleError::PoolTooLarge`], +/// which a large enough script can reach. +pub fn assemble(ops: &[Op]) -> Result<(Vec, Vec), AssembleError> { + let mut offsets = Vec::with_capacity(ops.len() + 1); + let mut at = 0u32; + for op in ops { + offsets.push(at); + at = at + .checked_add(encoded_width(op) as u32) + .ok_or(AssembleError::ChunkTooLarge { bytes: usize::MAX })?; + } + // One past the end, so a jump to "after the last instruction" resolves. + offsets.push(at); + + let mut code = Vec::with_capacity(at as usize); + for (index, op) in ops.iter().enumerate() { + let target = |target: u32| -> Result { + offsets + .get(target as usize) + .copied() + .ok_or(AssembleError::JumpOutOfRange { at: index, target }) + }; + + let small = |value: usize, what: &'static str| -> Result { + u16::try_from(value).map_err(|_| AssembleError::PoolTooLarge { + what, + entries: value, + }) + }; + + match op { + Op::Const(index) => { + code.push(tag::CONST); + code.extend_from_slice(&small(*index as usize, "constants")?.to_le_bytes()); + } + Op::Unit => code.push(tag::UNIT), + Op::Bool(false) => code.push(tag::FALSE), + Op::Bool(true) => code.push(tag::TRUE), + + Op::LoadLocal(slot) => { + code.push(tag::LOAD_LOCAL); + code.extend_from_slice(&slot.to_le_bytes()); + } + Op::StoreLocal(slot) => { + code.push(tag::STORE_LOCAL); + code.extend_from_slice(&slot.to_le_bytes()); + } + + Op::AssignLocal { + slot, + var_name, + op: None, + } => { + code.push(tag::ASSIGN_LOCAL); + code.extend_from_slice(&slot.to_le_bytes()); + code.extend_from_slice(&small(*var_name as usize, "names")?.to_le_bytes()); + } + Op::AssignLocal { + slot, + var_name, + op: Some(assign_op), + } => { + code.push(tag::ASSIGN_LOCAL_OP); + code.extend_from_slice(&slot.to_le_bytes()); + code.extend_from_slice(&small(*var_name as usize, "names")?.to_le_bytes()); + code.extend_from_slice( + &small(*assign_op as usize, "op-assignments")?.to_le_bytes(), + ); + } + + Op::LoadThis => code.push(tag::LOAD_THIS), + Op::LoadThisShared => code.push(tag::LOAD_THIS_SHARED), + Op::RequireThis => code.push(tag::REQUIRE_THIS), + Op::AssignThis { op: None } => code.push(tag::ASSIGN_THIS), + Op::AssignThis { + op: Some(assign_op), + } => { + code.push(tag::ASSIGN_THIS_OP); + code.extend_from_slice( + &small(*assign_op as usize, "op-assignments")?.to_le_bytes(), + ); + } + + Op::LoadNamed(name) => { + code.push(tag::LOAD_NAMED); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + + Op::AssignNamed { name, op: None } => { + code.push(tag::ASSIGN_NAMED); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + Op::AssignNamed { + name, + op: Some(assign_op), + } => { + code.push(tag::ASSIGN_NAMED_OP); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + code.extend_from_slice( + &small(*assign_op as usize, "op-assignments")?.to_le_bytes(), + ); + } + + Op::DeclareLocal { name, is_const } => { + code.push(if *is_const { + tag::DECLARE_CONST + } else { + tag::DECLARE_LOCAL + }); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + + Op::Pop => code.push(tag::POP), + + Op::Jump(to) => { + code.push(tag::JUMP); + code.extend_from_slice(&target(*to)?.to_le_bytes()); + } + Op::JumpIfTrue { target: to } => { + code.push(tag::JUMP_IF_TRUE); + code.extend_from_slice(&target(*to)?.to_le_bytes()); + } + Op::JumpIfFalse { target: to } => { + code.push(tag::JUMP_IF_FALSE); + code.extend_from_slice(&target(*to)?.to_le_bytes()); + } + + Op::Call { + name, + argc, + op: None, + } => { + code.push(tag::CALL); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + code.push(*argc); + } + Op::Call { + name, + argc, + op: Some(token), + } => { + code.push(tag::CALL_OP); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + code.push(*argc); + code.extend_from_slice(&small(*token as usize, "operators")?.to_le_bytes()); + } + + Op::CallRef { + name, + argc, + receiver, + } => { + // `this` is a register and needs no operand to address it, so + // its encoding is the other two minus the trailing `u16`. + let operand = match receiver { + Receiver::Local(slot) => Some((tag::CALL_LOCAL_REF, *slot)), + Receiver::Named(var) => { + Some((tag::CALL_NAMED_REF, small(*var as usize, "names")?)) + } + Receiver::This => None, + }; + code.push(operand.map_or(tag::CALL_THIS_REF, |(tag, _)| tag)); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + code.push(*argc); + if let Some((_, operand)) = operand { + code.extend_from_slice(&operand.to_le_bytes()); + } + } + + Op::Rotate(under) => { + code.push(tag::ROTATE); + code.push(*under); + } + + Op::Chain(index) => { + code.push(tag::CHAIN); + code.extend_from_slice(&small(*index as usize, "chains")?.to_le_bytes()); + } + + Op::MakeArray(len) => { + code.push(tag::MAKE_ARRAY); + code.extend_from_slice(&len.to_le_bytes()); + } + + Op::MakeMap(len) => { + code.push(tag::MAKE_MAP); + code.extend_from_slice(&len.to_le_bytes()); + } + + Op::CheckSize { index, map } => { + code.push(if *map { + tag::CHECK_MAP_SIZE + } else { + tag::CHECK_ARRAY_SIZE + }); + code.extend_from_slice(&index.to_le_bytes()); + } + + Op::Share(slot) => { + code.push(tag::SHARE); + code.extend_from_slice(&slot.to_le_bytes()); + } + Op::ShareNamed(name) => { + code.push(tag::SHARE_NAMED); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + Op::LoadShared(slot) => { + code.push(tag::LOAD_SHARED); + code.extend_from_slice(&slot.to_le_bytes()); + } + Op::LoadSharedNamed(name) => { + code.push(tag::LOAD_SHARED_NAMED); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + + Op::MakeClosure(name) => { + code.push(tag::MAKE_CLOSURE); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + Op::MakeFnPtr => code.push(tag::MAKE_FN_PTR), + Op::IsShared => code.push(tag::IS_SHARED), + Op::Curry(argc) => { + code.push(tag::CURRY); + code.push(*argc); + } + Op::CallFnPtr { + argc, + method, + receiver, + } => { + // The receiver's value is on the stack whichever of these it + // is; the tag says where it came from, and two of them carry + // enough to reach it again. + let operand = match receiver { + Some(Receiver::Local(slot)) => Some((tag::CALL_FN_PTR_ON_LOCAL, *slot)), + Some(Receiver::Named(var)) => { + Some((tag::CALL_FN_PTR_ON_NAMED, small(*var as usize, "names")?)) + } + Some(Receiver::This) => Some((tag::CALL_FN_PTR_ON_THIS, 0)), + None if *method => Some((tag::CALL_FN_PTR_METHOD, 0)), + None => Some((tag::CALL_FN_PTR, 0)), + }; + let (tag, operand) = operand.expect("every arm answers"); + code.push(tag); + code.push(*argc); + if matches!(tag, tag::CALL_FN_PTR_ON_LOCAL | tag::CALL_FN_PTR_ON_NAMED) { + code.extend_from_slice(&operand.to_le_bytes()); + } + } + + Op::InterpolateStart => code.push(tag::INTERPOLATE_START), + Op::InterpolateAppend => code.push(tag::INTERPOLATE_APPEND), + Op::InterpolateEnd => code.push(tag::INTERPOLATE_END), + + // The table's own targets are instruction indices too, but they + // are not in the code — see `resolve_switch_targets`. + Op::Switch(index) => { + code.push(tag::SWITCH); + code.extend_from_slice(&small(*index as usize, "switches")?.to_le_bytes()); + } + + Op::UnwindTo(depth) => { + code.push(tag::UNWIND_TO); + code.extend_from_slice(&depth.to_le_bytes()); + } + + Op::Tick => code.push(tag::TICK), + Op::Checkpoint => code.push(tag::CHECKPOINT), + Op::Throw => code.push(tag::THROW), + Op::IterInit => code.push(tag::ITER_INIT), + Op::IterDrop => code.push(tag::ITER_DROP), + Op::PopHandler => code.push(tag::POP_HANDLER), + + Op::PushHandler { + target: to, + catch_var, + } => { + code.push(match catch_var { + Some(..) => tag::PUSH_HANDLER_VAR, + None => tag::PUSH_HANDLER, + }); + code.extend_from_slice(&target(*to)?.to_le_bytes()); + if let Some(name) = catch_var { + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + } + + Op::IterNext { exit, indexed } => { + code.push(if *indexed { + tag::ITER_NEXT_INDEXED + } else { + tag::ITER_NEXT + }); + code.extend_from_slice(&target(*exit)?.to_le_bytes()); + } + + Op::StoreShared(slot) => { + code.push(tag::STORE_SHARED); + code.extend_from_slice(&slot.to_le_bytes()); + } + Op::Return => code.push(tag::RETURN), + + Op::EvalAst { + residual, + rewind_scope, + } => { + code.push(if *rewind_scope { + tag::EVAL_AST + } else { + tag::EVAL_AST_KEEP + }); + code.extend_from_slice(&small(*residual as usize, "fragments")?.to_le_bytes()); + } + } + } + + Ok((code, offsets)) +} + +/// Rewrite switch targets from instruction indices into byte offsets. +/// +/// The other half of [`assemble`], and separate only because a table is not in +/// the instruction stream: [`Op::Switch`] carries a pool index, and the jumps +/// are inside the pool entry. Same `offsets` table, same one-past-the-end +/// entry, so a `switch` whose default is the end of the chunk resolves. +/// +/// # Errors +/// +/// [`AssembleError::SwitchTargetOutOfRange`] for a target naming no +/// instruction, which is a compiler bug. +pub fn resolve_switch_targets( + switches: &mut [super::Switch], + offsets: &[u32], +) -> Result<(), AssembleError> { + for (table, switch) in switches.iter_mut().enumerate() { + let resolve = |target: &mut u32| -> Result<(), AssembleError> { + *target = + *offsets + .get(*target as usize) + .ok_or(AssembleError::SwitchTargetOutOfRange { + table, + target: *target, + })?; + Ok(()) + }; + + for case in &mut switch.cases { + resolve(&mut case.target)?; + } + for range in &mut switch.ranges { + resolve(&mut range.target)?; + } + resolve(&mut switch.default)?; + } + Ok(()) +} + +/// How many bytes an instruction will take once written. +fn encoded_width(op: &Op) -> usize { + match op { + Op::Unit + | Op::Bool(..) + | Op::Pop + | Op::Tick + | Op::Checkpoint + | Op::Throw + | Op::IterInit + | Op::IterDrop + | Op::PopHandler + | Op::InterpolateStart + | Op::InterpolateAppend + | Op::InterpolateEnd + | Op::MakeFnPtr + | Op::IsShared + | Op::LoadThis + | Op::LoadThisShared + | Op::RequireThis + | Op::AssignThis { op: None } + | Op::Return => 1, + Op::Curry(..) | Op::Rotate(..) => 2, + Op::CallFnPtr { receiver, .. } => match receiver { + Some(Receiver::Local(..) | Receiver::Named(..)) => 4, + Some(Receiver::This) | None => 2, + }, + Op::Const(..) + | Op::LoadLocal(..) + | Op::StoreLocal(..) + | Op::DeclareLocal { .. } + | Op::UnwindTo(..) + | Op::EvalAst { .. } + | Op::Chain(..) + | Op::Switch(..) + | Op::LoadNamed(..) + | Op::AssignNamed { op: None, .. } + | Op::StoreShared(..) + | Op::Share(..) + | Op::ShareNamed(..) + | Op::LoadShared(..) + | Op::LoadSharedNamed(..) + | Op::MakeClosure(..) + | Op::MakeArray(..) + | Op::MakeMap(..) + | Op::AssignThis { op: Some(..) } + | Op::CheckSize { .. } => 3, + Op::Call { op: None, .. } + | Op::CallRef { + receiver: Receiver::This, + .. + } => 4, + Op::AssignLocal { op: None, .. } + | Op::AssignNamed { op: Some(..), .. } + | Op::Jump(..) + | Op::JumpIfTrue { .. } + | Op::JumpIfFalse { .. } + | Op::IterNext { .. } + | Op::PushHandler { + catch_var: None, .. + } => 5, + Op::PushHandler { + catch_var: Some(..), + .. + } => 7, + Op::Call { op: Some(..), .. } + | Op::CallRef { + receiver: Receiver::Local(..) | Receiver::Named(..), + .. + } => 6, + Op::AssignLocal { op: Some(..), .. } => 7, + } +} + +/// Recover the instruction at `at`, for disassembly and tests. +/// +/// Jump targets come back as byte offsets, not the instruction indices the +/// compiler used, because that is what the code actually holds. +#[must_use] +pub fn decode(code: &[u8], at: usize) -> Option { + width(code, at)?; + let small = |offset: usize| u16_at(code, at + offset); + + Some(match code[at] { + tag::CONST => Op::Const(u32::from(small(1)?)), + tag::UNIT => Op::Unit, + tag::FALSE => Op::Bool(false), + tag::TRUE => Op::Bool(true), + + tag::LOAD_LOCAL => Op::LoadLocal(small(1)?), + tag::STORE_LOCAL => Op::StoreLocal(small(1)?), + + tag::ASSIGN_LOCAL => Op::AssignLocal { + slot: small(1)?, + var_name: u32::from(small(3)?), + op: None, + }, + tag::ASSIGN_LOCAL_OP => Op::AssignLocal { + slot: small(1)?, + var_name: u32::from(small(3)?), + op: Some(u32::from(small(5)?)), + }, + + tag::LOAD_NAMED => Op::LoadNamed(u32::from(small(1)?)), + tag::ASSIGN_NAMED => Op::AssignNamed { + name: u32::from(small(1)?), + op: None, + }, + tag::ASSIGN_NAMED_OP => Op::AssignNamed { + name: u32::from(small(1)?), + op: Some(u32::from(small(3)?)), + }, + + tag::LOAD_THIS => Op::LoadThis, + tag::LOAD_THIS_SHARED => Op::LoadThisShared, + tag::REQUIRE_THIS => Op::RequireThis, + tag::ASSIGN_THIS => Op::AssignThis { op: None }, + tag::ASSIGN_THIS_OP => Op::AssignThis { + op: Some(u32::from(small(1)?)), + }, + + tag::DECLARE_LOCAL => Op::DeclareLocal { + name: u32::from(small(1)?), + is_const: false, + }, + tag::DECLARE_CONST => Op::DeclareLocal { + name: u32::from(small(1)?), + is_const: true, + }, + + tag::POP => Op::Pop, + + tag::JUMP => Op::Jump(u32_at(code, at + 1)?), + tag::JUMP_IF_TRUE => Op::JumpIfTrue { + target: u32_at(code, at + 1)?, + }, + tag::JUMP_IF_FALSE => Op::JumpIfFalse { + target: u32_at(code, at + 1)?, + }, + + tag::CALL => Op::Call { + name: u32::from(small(1)?), + argc: code[at + 3], + op: None, + }, + tag::CALL_OP => Op::Call { + name: u32::from(small(1)?), + argc: code[at + 3], + op: Some(u32::from(small(4)?)), + }, + + tag::CALL_LOCAL_REF => Op::CallRef { + name: u32::from(small(1)?), + argc: code[at + 3], + receiver: Receiver::Local(small(4)?), + }, + tag::CALL_THIS_REF => Op::CallRef { + name: u32::from(small(1)?), + argc: code[at + 3], + receiver: Receiver::This, + }, + tag::CALL_NAMED_REF => Op::CallRef { + name: u32::from(small(1)?), + argc: code[at + 3], + receiver: Receiver::Named(u32::from(small(4)?)), + }, + tag::ROTATE => Op::Rotate(code[at + 1]), + + tag::CHAIN => Op::Chain(u32::from(small(1)?)), + tag::SWITCH => Op::Switch(u32::from(small(1)?)), + tag::MAKE_ARRAY => Op::MakeArray(small(1)?), + tag::MAKE_MAP => Op::MakeMap(small(1)?), + tag::CHECK_ARRAY_SIZE => Op::CheckSize { + index: small(1)?, + map: false, + }, + tag::CHECK_MAP_SIZE => Op::CheckSize { + index: small(1)?, + map: true, + }, + tag::SHARE => Op::Share(small(1)?), + tag::SHARE_NAMED => Op::ShareNamed(u32::from(small(1)?)), + tag::LOAD_SHARED => Op::LoadShared(small(1)?), + tag::LOAD_SHARED_NAMED => Op::LoadSharedNamed(u32::from(small(1)?)), + tag::MAKE_CLOSURE => Op::MakeClosure(u32::from(small(1)?)), + tag::MAKE_FN_PTR => Op::MakeFnPtr, + tag::IS_SHARED => Op::IsShared, + tag::CURRY => Op::Curry(code[at + 1]), + tag::CALL_FN_PTR => Op::CallFnPtr { + argc: code[at + 1], + method: false, + receiver: None, + }, + tag::CALL_FN_PTR_METHOD => Op::CallFnPtr { + argc: code[at + 1], + method: true, + receiver: None, + }, + tag::CALL_FN_PTR_ON_LOCAL => Op::CallFnPtr { + argc: code[at + 1], + method: true, + receiver: Some(Receiver::Local(small(2)?)), + }, + tag::CALL_FN_PTR_ON_NAMED => Op::CallFnPtr { + argc: code[at + 1], + method: true, + receiver: Some(Receiver::Named(u32::from(small(2)?))), + }, + tag::CALL_FN_PTR_ON_THIS => Op::CallFnPtr { + argc: code[at + 1], + method: true, + receiver: Some(Receiver::This), + }, + tag::INTERPOLATE_START => Op::InterpolateStart, + tag::INTERPOLATE_APPEND => Op::InterpolateAppend, + tag::INTERPOLATE_END => Op::InterpolateEnd, + tag::UNWIND_TO => Op::UnwindTo(small(1)?), + tag::TICK => Op::Tick, + tag::CHECKPOINT => Op::Checkpoint, + tag::THROW => Op::Throw, + tag::ITER_INIT => Op::IterInit, + tag::ITER_DROP => Op::IterDrop, + tag::ITER_NEXT => Op::IterNext { + exit: u32_at(code, at + 1)?, + indexed: false, + }, + tag::ITER_NEXT_INDEXED => Op::IterNext { + exit: u32_at(code, at + 1)?, + indexed: true, + }, + tag::STORE_SHARED => Op::StoreShared(small(1)?), + tag::POP_HANDLER => Op::PopHandler, + tag::PUSH_HANDLER => Op::PushHandler { + target: u32_at(code, at + 1)?, + catch_var: None, + }, + tag::PUSH_HANDLER_VAR => Op::PushHandler { + target: u32_at(code, at + 1)?, + catch_var: Some(u32::from(small(5)?)), + }, + tag::RETURN => Op::Return, + + tag::EVAL_AST => Op::EvalAst { + residual: u32::from(small(1)?), + rewind_scope: true, + }, + tag::EVAL_AST_KEEP => Op::EvalAst { + residual: u32::from(small(1)?), + rewind_scope: false, + }, + + _ => return None, + }) +} + +/// Every instruction in a chunk, paired with its address. +/// +/// Stops at the first thing it cannot decode, so it is safe to point at +/// anything. For a chunk that verified, it reaches the end. +pub fn disassemble(code: &[u8]) -> impl Iterator + '_ { + let mut at = 0usize; + core::iter::from_fn(move || { + let op = decode(code, at)?; + let here = at; + at += width(code, at)?; + Some((here, op)) + }) +} + +/// A chunk's instructions, owned when compiled and borrowed when loaded. +/// +/// Borrowing is the point. A program read from an artifact holds a slice of +/// those bytes and allocates nothing for its code — which is the difference +/// between retaining sixteen bytes an instruction and retaining three. +pub type Code<'a> = Cow<'a, [u8]>; + +#[cfg(test)] +mod tests { + use super::*; + + /// The property everything else rests on: what the compiler emitted is what + /// comes back, with jumps rewritten from indices to the addresses those + /// instructions actually landed at. + #[test] + fn instructions_survive_assembly() { + let ops = vec![ + Op::Const(7), + Op::Unit, + Op::Bool(true), + Op::Bool(false), + Op::LoadLocal(3), + Op::StoreLocal(4), + Op::AssignLocal { + slot: 1, + var_name: 2, + op: None, + }, + Op::AssignLocal { + slot: 1, + var_name: 2, + op: Some(5), + }, + Op::DeclareLocal { + name: 8, + is_const: false, + }, + Op::DeclareLocal { + name: 9, + is_const: true, + }, + Op::Pop, + Op::Call { + name: 1, + argc: 2, + op: None, + }, + Op::Call { + name: 1, + argc: 2, + op: Some(3), + }, + Op::CallRef { + name: 1, + argc: 2, + receiver: Receiver::Local(4), + }, + Op::CallRef { + name: 1, + argc: 2, + receiver: Receiver::Named(5), + }, + Op::CallRef { + name: 1, + argc: 2, + receiver: Receiver::This, + }, + Op::CallFnPtr { + argc: 1, + method: false, + receiver: None, + }, + Op::CallFnPtr { + argc: 1, + method: true, + receiver: None, + }, + Op::CallFnPtr { + argc: 1, + method: true, + receiver: Some(Receiver::Local(4)), + }, + Op::CallFnPtr { + argc: 1, + method: true, + receiver: Some(Receiver::Named(5)), + }, + Op::CallFnPtr { + argc: 1, + method: true, + receiver: Some(Receiver::This), + }, + Op::LoadThis, + Op::LoadThisShared, + Op::RequireThis, + Op::AssignThis { op: None }, + Op::AssignThis { op: Some(6) }, + Op::Rotate(3), + Op::UnwindTo(6), + Op::Tick, + Op::EvalAst { + residual: 0, + rewind_scope: true, + }, + Op::EvalAst { + residual: 1, + rewind_scope: false, + }, + Op::Return, + ]; + + let (code, offsets) = assemble(&ops).expect("must assemble"); + let back: Vec<_> = disassemble(&code).map(|(_, op)| op).collect(); + + assert_eq!(back, ops); + assert_eq!(offsets.len(), ops.len() + 1); + assert_eq!( + *offsets.last().unwrap() as usize, + code.len(), + "the trailing offset must be the end of the chunk", + ); + } + + #[test] + fn jumps_become_the_addresses_of_the_instructions_they_named() { + // Index 3 is `Return`, which lands after Const(3) + Unit + Pop. + let ops = vec![Op::Const(0), Op::Unit, Op::Pop, Op::Return, Op::Jump(3)]; + let (code, offsets) = assemble(&ops).expect("must assemble"); + + assert_eq!(offsets[3], 3 + 1 + 1); + assert_eq!(decode(&code, offsets[4] as usize), Some(Op::Jump(5))); + } + + /// The compiler emits a jump to "one past the last instruction" when a + /// block's exit is the end of the chunk. + #[test] + fn a_jump_past_the_last_instruction_resolves_to_the_end() { + let ops = vec![Op::Unit, Op::Jump(2)]; + let (code, _) = assemble(&ops).expect("must assemble"); + assert_eq!(decode(&code, 1), Some(Op::Jump(code.len() as u32))); + } + + #[test] + fn a_jump_to_an_instruction_that_does_not_exist_is_refused() { + assert_eq!( + assemble(&[Op::Jump(99)]), + Err(AssembleError::JumpOutOfRange { at: 0, target: 99 }), + ); + } + + /// Operand widths are the format's hard limit, and the compiler falls back + /// rather than writing a truncated index. + #[test] + fn a_pool_index_too_wide_for_its_operand_is_refused() { + assert_eq!( + assemble(&[Op::Const(70_000)]), + Err(AssembleError::PoolTooLarge { + what: "constants", + entries: 70_000, + }), + ); + } + + #[test] + fn an_unknown_tag_has_no_width_and_does_not_decode() { + assert_eq!(width(&[0xff, 0, 0], 0), None); + assert_eq!(decode(&[0xff, 0, 0], 0), None); + } + + /// A truncated operand must not read whatever follows in memory. + #[test] + fn an_instruction_cut_short_does_not_decode() { + assert_eq!( + width(&[tag::CONST, 0], 0), + None, + "one byte of a u16 operand" + ); + assert_eq!(decode(&[tag::CONST, 0], 0), None); + assert_eq!(decode(&[tag::JUMP, 0, 0], 0), None); + } + + /// Disassembly is pointed at untrusted bytes by `dump`, so it stops rather + /// than running away. + #[test] + fn disassembling_junk_terminates() { + let junk = [0xff; 32]; + assert_eq!(disassemble(&junk).count(), 0); + + let partly_good = [tag::UNIT, tag::POP, 0xff, tag::UNIT]; + assert_eq!(disassemble(&partly_good).count(), 2); + } +} diff --git a/src/grain/bytecode/mod.rs b/src/grain/bytecode/mod.rs new file mode 100644 index 000000000..d34b6d3d3 --- /dev/null +++ b/src/grain/bytecode/mod.rs @@ -0,0 +1,21 @@ +//! The instruction set, the pools it indexes, and the checks a chunk must pass +//! before it runs. + +pub mod code; + +mod chain; +mod chunk; +mod op; +mod positions; +mod strings; +mod switch; +mod verify; + +pub use chain::{Chain, Root, Step, Tail}; +pub use chunk::Chunk; +pub use code::{assemble, disassemble, resolve_switch_targets, AssembleError, Code}; +pub use op::{AssignOp, Op, Receiver}; +pub use positions::{Positions, TableError}; +pub use strings::{BadTable, Strings}; +pub use switch::{probe, Switch, SwitchCase, SwitchRange}; +pub use verify::{verify, Pools, VerifyError}; diff --git a/src/grain/bytecode/op.rs b/src/grain/bytecode/op.rs new file mode 100644 index 000000000..e7498d863 --- /dev/null +++ b/src/grain/bytecode/op.rs @@ -0,0 +1,609 @@ +use crate::tokenizer::Token; + +/// What `x op= y` needs to reproduce rhai's resolution order. +/// +/// Both the op-assignment and the plain operator are carried, because rhai +/// tries the first and falls back to expanding into the second when no +/// op-assignment implementation exists (`eval/stmt.rs:217-236`). +/// +/// Lives in the program's op-assignment pool rather than in the instruction: +/// four fields including two `Token`s do not fit an operand, and the same +/// `+=` used in ten places is one entry. +// No `Eq`: `Token` carries float literals, so it is only `PartialEq`. +#[derive(Debug, Clone, PartialEq)] +pub struct AssignOp { + /// The `+=` token, for the built-in lookup. + pub op_assign: Token, + /// `"+="`, for dispatch and for error messages. + pub op_assign_name: u32, + /// The `+` token, for the expansion. + pub op: Token, + /// `"+"`. + pub op_name: u32, +} + +/// Where [`Op::CallRef`] finds the variable it calls through. +/// +/// The two differ in how the variable is reached, not in what happens to it: +/// both take a reference where rhai would and fall back to a value where it +/// would not, by the same rule. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Receiver { + /// A local, addressed by slot. Nothing was pushed for it — the call reads + /// the scope entry itself, and a slot always names one. + Local(u16), + + /// A variable no slot addresses: the caller's, a module's, or nothing. + /// + /// [`Op::LoadNamed`] has already resolved the name and left its value as + /// argument zero, which is what raises `ErrorVariableNotFound` against the + /// variable rather than against the call — two positions the table cannot + /// give one instruction. The call re-reaches the scope entry for the + /// reference and falls back to that value when there is no entry to reach: + /// a resolver's answer, a module's constant, a `const`. + /// + /// So the by-reference path pays for a clone it discards. Worth removing + /// only if a profile of a host-heavy script says so; a local, which is the + /// common receiver by far, never makes one. + Named(u32), + + /// The frame's receiver, for `f(this, ..)`. + /// + /// Rhai applies the same rewrite to `this` as to a variable, but only when + /// the receiver is neither shared nor curried (`func/call.rs:1409-1433`). + /// Shared-ness is a run-time property, so the value arrives on the stack as + /// argument zero and the call reaches for the register instead when it turns + /// out to be usable by reference — the deferral [`Receiver::Local`] already + /// makes for a read-only entry. + /// + /// Unlike either of the others, [`Op::LoadThis`] pushes it *before* the + /// remaining arguments. Rhai's two arms disagree about when `this` is read: + /// the by-reference one takes it after them (`func/call.rs:1417`), but the + /// fallback that a shared or unbound receiver lands in reads and flattens it + /// first (`:1462`). Reading first is what makes `f(this, { this = 9; 1 })` + /// pass the pre-mutation value, and an unbound `f(this, nosuch)` report + /// `ErrorUnboundThis` rather than `ErrorVariableNotFound`. + This, +} + +/// One VM instruction, as the compiler emits it and a disassembly shows it. +/// +/// **Not the executed form.** A program's code is a byte slice, assembled from +/// these by [`assemble`](crate::grain::bytecode::assemble) and dispatched on +/// directly, so a loaded program can borrow its instructions from the artifact +/// rather than building sixteen bytes of enum per instruction. See +/// [`code`](crate::grain::bytecode::code) for the encoding. +/// +/// A stack machine: operands are pushed and consumed on an operand stack, and +/// locals live in slots addressed directly. `EvalAst` is the escape hatch that +/// hands a fragment back to rhai's tree walker, so anything the compiler cannot +/// yet lower still runs, and the whole language stays covered. Lowering more of +/// it converts residuals into instructions rather than adding coverage. +/// +/// Instructions carry no source position. Several of them can fail against a +/// place in the source, and the position for that comes from the program's +/// [`Positions`](crate::grain::bytecode::Positions) table, keyed on the instruction's +/// own address. Keeping it out means the diagnostics can be stripped from an +/// artifact without touching the code. +/// +/// Anything too wide for an operand is a `u32` index into one of the program's +/// pools, which is also what keeps a repeated operator or name from being +/// stored twice. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Op { + /// Push constant `.0` from the pool. + Const(u32), + /// Push unit. + Unit, + /// Push a boolean. + Bool(bool), + + /// Push the value in local slot `.0`. + LoadLocal(u16), + /// Pop and write into local slot `.0`, which must already exist. + StoreLocal(u16), + + /// Push the value of the variable named `.0`, found by name. + /// + /// For the variables no slot can address: the ones the caller already had + /// in its `Scope` when the program started, which sit below the base every + /// slot is measured from. Without this, a script that reads anything its + /// host supplied is a fragment, and so cannot be written to an artifact at + /// all. + /// + /// Three places are searched, in rhai's order (`eval/expr.rs:107-155`): + /// the resolver a host may have registered with `Engine::on_var`, then the + /// scope, then the modules loaded into the global namespace. Missing from + /// all three is `ErrorVariableNotFound`. + /// + /// A reverse scan of the scope per read, where a slot is an index — which + /// is why the compiler only emits this for a name it could not resolve. + LoadNamed(u32), + + /// Pop a value and assign it to the variable named `name`, optionally + /// through an operator. + /// + /// [`Op::LoadNamed`]'s counterpart, and resolved the same way. Assigning + /// to anything that is not a scope entry it can take a reference to — a + /// value the resolver produced, a module's constant, a `const` — is + /// `ErrorAssignmentToConstant`, as it is in the walker. + AssignNamed { + /// The name of the variable + name: u32, + /// Index into the op-assignment pool; absent for a plain `=`. + op: Option, + }, + + /// Pop a value and assign it to local slot `slot`, optionally through an + /// operator. + /// + /// Separate from `StoreLocal` because `x += y` is not `x = x + y`: rhai + /// looks for an op-assignment implementation that mutates in place, and + /// only expands to the binary form if there is none. + AssignLocal { + /// The slot index + slot: u16, + /// Names the variable in `ErrorAssignmentToConstant`. + var_name: u32, + /// Index into the op-assignment pool; absent for a plain `=`. + op: Option, + }, + /// Pop and declare it as a new local, extending the scope by one. + /// + /// Slots are assigned in declaration order, so the new local always lands + /// at the top of the scope. Carries the name because locals live in the + /// caller's `Scope`, where entries are named, and carries constness + /// because rhai enforces it through the value's own access mode. + DeclareLocal { + /// The name of the variable + name: u32, + /// Whether the variable is declared `const`. + is_const: bool, + }, + + /// Discard the top of the operand stack. + Pop, + + /// Jump to `.0`. + /// + /// An instruction index as the compiler emits it, a byte offset once + /// assembled — instructions vary in length, so there is nothing else it + /// could be. + Jump(u32), + /// Pop a condition and jump to `.0` if it is true. Mirrors + /// [`Op::JumpIfFalse`]; both exist so short-circuit `&&` and `||` lower + /// without an extra negation. + JumpIfTrue { + /// Where to jump to + target: u32, + }, + /// Pop a condition and jump to `.0` if it is false. + /// + /// Its position-table entry is the condition's own position, because rhai + /// rejects a non-boolean guard against the guard expression rather than the + /// statement — and the differential harness compares error positions. + JumpIfFalse { + /// Where to jump to + target: u32, + }, + + /// Pop `argc` arguments and call the function named by `name`, pushing the + /// result. + /// + /// Dispatch goes through rhai, so every registered function, operator and + /// script function resolves exactly as it would in the walker. Only calls + /// rhai handles syntactically before dispatch — `Fn`, `call`, `curry`, + /// `eval`, `is_def_var` — are excluded, and stay fragments. + /// + /// The position table's entry for this instruction is the call site. Rhai's + /// dispatch path takes one and reports failures against it; `call_fn_raw` + /// does not, so an error that comes back without a position gets this one. + /// + /// `op` indexes the operator pool when the call is an operator, and names + /// the token the built-in lookup keys on. The walker short-circuits these + /// to a function pointer rather than dispatching, and a VM that did not + /// would be slower than the tree it replaced. + Call { + /// The name of the function + name: u32, + /// How many arguments to pop + argc: u8, + /// Index into the operator pool; absent unless the call is an operator. + op: Option, + }, + + /// Call `name` with a variable as its first argument, taken by reference. + /// + /// Rhai rewrites `f(x, ..)` into `x.f(..)` whenever the first argument is a + /// plain variable, so that a `&mut` first parameter mutates the variable + /// rather than a copy (`func/call.rs:1434-1460`). `push(a, 2)` and + /// `a.push(2)` are the same call; only the second reached the mutation + /// through [`Op::Chain`]. + /// + /// Two things follow from the rewrite, and together they are why this is an + /// instruction rather than an argument order: + /// + /// * the variable is read *after* the other arguments, so an argument that + /// writes to it is seen; + /// * a shared or read-only variable is passed by value instead — rhai hands + /// out a reference to neither (`func/call.rs:1449-1454`). + /// + /// Operators never reach here: under `fast_operators` a binary one + /// short-circuits before the rewrite (`func/call.rs:1775`), so `a + b` + /// reads `a` first and needs no reference. + CallRef { + /// The name of the function + name: u32, + /// How many arguments to pop, not counting the receiver. + argc: u8, + /// Where the first argument is found. + receiver: Receiver, + }, + + /// Move the top of the operand stack down past `.0` values. + /// + /// A [`Receiver::Named`] receiver is resolved by [`Op::LoadNamed`] after + /// the other arguments, and this puts it back in argument order. + Rotate(u8), + + /// Pop a subject and jump to wherever switch table `.0` sends it. + /// + /// Always jumps — the table's default is where a subject that matches + /// nothing goes, and an absent `_` arm compiles to a jump past the + /// statement. Arms with guards are not table entries: the table sends a + /// subject to the head of a chain that tries each guard in source order + /// and falls through to the default, which is what keeps dispatch a + /// lookup. + /// + /// The table is in the program's switch pool rather than in the + /// instruction because it is unbounded, and because two arms of one + /// `switch` share it. + Switch(u32), + + /// Turn local slot `.0` into a shared cell, so a closure can capture it. + /// + /// Rhai's parser emits one of these per captured variable ahead of the + /// `curry` call that binds them (`parser.rs:3707`). Sharing is what makes + /// the closure and the enclosing scope see the same value afterwards; the + /// write-through in `place` is the other half. + /// + /// The variable resolver gets first refusal, as it does in + /// `eval/stmt.rs:998`: if a host's `on_var` answers the name, the variable + /// is *not* shared. + Share(u16), + + /// The same for a variable no slot names — one the caller supplied. + ShareNamed(u32), + + /// Push local slot `.0` without flattening it. + /// + /// A read normally hands back what a shared cell contains, which is right + /// for a value and wrong for a capture: currying a closure has to bind the + /// *cell*, or the closure gets a copy and stops being a closure. + LoadShared(u16), + + /// The same for a variable no slot names — one the caller supplied. + /// + /// [`Op::LoadNamed`] flattens, and a closure capturing a caller's variable + /// through that read binds a copy: the aliasing is dead, and a write to the + /// variable afterwards is invisible to the closure. The slot case has had + /// [`Op::LoadShared`] since closures were lowered at all; this is the half + /// that was missing. + LoadSharedNamed(u32), + + /// Push the receiver bound to the running frame. + /// + /// `this` is not a scope entry and no slot addresses it: rhai threads it + /// through evaluation as a parameter (`func/script.rs:29`) and keeps it out + /// of the `Scope` altogether. So it gets a register of its own, and these + /// four instructions are the only things that reach it. + /// + /// Flattens, as [`Op::LoadLocal`] does. Rhai reads `this` *unflattened* + /// (`eval/expr.rs:272`) but flattens at almost every consumer — a `let` + /// (`eval/stmt.rs:436`), an assignment's right-hand side (`:321`), a call's + /// arguments (`func/call.rs:1428`) — so the flattening read is the common + /// one and [`Op::LoadThisShared`] is the exception, exactly as it is for a + /// local. + /// + /// `ErrorUnboundThis` when the frame has no receiver. + LoadThis, + + /// The same without flattening. + /// + /// [`Op::LoadShared`]'s counterpart, for the three readers that have to see + /// the cell rather than what it holds: a `switch` subject, `is_shared`, and + /// a curried capture. + LoadThisShared, + + /// Raise `ErrorUnboundThis` if the frame has no receiver. Pushes nothing. + /// + /// `this = v` checks *before* it evaluates `v` (`eval/stmt.rs:299-302`), + /// unlike the variable arm, which evaluates the value first (`:319-323`). + /// Without a check of its own, `this = nosuch` in an unbound frame would + /// report `ErrorVariableNotFound` where rhai reports `ErrorUnboundThis`. + RequireThis, + + /// Pop a value and assign it to the frame's receiver, optionally through an + /// operator. + /// + /// [`Op::AssignLocal`] without the slot or the name, because `this` has + /// neither — and neither does rhai's own failure here: assigning to a + /// read-only receiver is `ErrorAssignmentToConstant("")` + /// (`eval/stmt.rs:118-122`), named for an expression that has no name. + AssignThis { + /// Index into the op-assignment pool; absent for a plain `=`. + op: Option, + }, + + /// Pop a value and push whether it is a shared cell. + /// + /// Rhai answers `is_shared` before dispatch and registers no function for + /// it (`func/call.rs:1240`), so there is nothing to call — and the value + /// has to arrive unflattened, or the answer is always false. + IsShared, + + /// Push a function pointer to the compiled function named `.0`. + /// + /// A closure's, whose name the parser makes up (`anon$…`) and which + /// [`Op::MakeFnPtr`] would refuse — rhai only builds pointers to names a + /// script could have written. The name is known here, so unlike + /// `MakeFnPtr` it needs no operand on the stack. + MakeClosure(u32), + + /// Pop a name and push a function pointer to it. + /// + /// Deliberately the *late-bound* kind, carrying a name and nothing else. + /// Rhai's other kind embeds a `ScriptFuncDef` — an AST body — which is + /// both unreachable from outside the crate and exactly the allocation this + /// project exists to remove. Building our own means a pointer resolves + /// through the compiled function table like any other call, and a program + /// holding one can still be written to an artifact. + /// + /// The cost is that a `Normal` pointer is late-bound where rhai's is + /// early-bound: redefining the function after taking a pointer to it is + /// visible here and not in the walker. + MakeFnPtr, + + /// Pop `.0` arguments and a function pointer, and push the pointer with + /// those arguments bound to the front of it. + Curry(u8), + + /// Pop `argc` arguments and a target, and call a function pointer. + /// + /// A compiled function of that name and arity is called directly, with the + /// curried arguments spliced in front. Anything else — a native function, + /// a name that resolves elsewhere — goes to rhai's own `call_raw`. + /// + /// `method` distinguishes `f.call(x)` from `call(f, x)`, which are not the + /// same call. In method position a target that is *not* a pointer is not + /// an error: rhai takes the first argument as the pointer and binds the + /// target as `this` (`func/call.rs:816-919`), which is how a closure is + /// called against a receiver. + CallFnPtr { + /// How many arguments to pop + argc: u8, + /// Whether the call is in method position (`f.call(x)`). + method: bool, + /// Where the receiver came from, when there is anywhere to put it back. + /// + /// `obj.call(f)` binds `obj` as the closure's `this` **by reference** + /// (`func/call.rs:862`), so a closure that writes to `this` writes to + /// `obj`. The receiver's *value* is on the operand stack either way — + /// this only says where it came from, so the write can be carried back + /// there. + /// + /// `None` in call position, and for a receiver with nowhere to write + /// back to: `[1, 2].call(f)` mutates a temporary, as it does in rhai. + /// Only meaningful when `method` is set. + receiver: Option, + }, + + /// Push an empty buffer for an interpolated string to be built in. + /// + /// Interpolation is three instructions rather than one because rhai checks + /// the size limit after **every** segment and blames the segment that went + /// over. One instruction has one position-table entry, so it could not say + /// which; a pool of per-segment positions would say it but would not be + /// strippable, and diagnostics staying separable is the point of the + /// table. An instruction per segment puts each position exactly where the + /// rest of them live. + /// + /// The buffer is an ordinary operand, so a nested interpolation needs + /// nothing special. + InterpolateStart, + + /// Pop a segment and append it to the buffer beneath it. + /// + /// Not `+`, which is what it looks like: `+` is overridable and + /// interpolation is not, and the string-plus-anything operator skips this + /// size check. A string segment is written straight out and never reaches + /// dispatch; anything else goes through rhai's `to_string` rendering, + /// which consults native functions only. + InterpolateAppend, + + /// Replace the buffer with the interned string it built. + InterpolateEnd, + + /// Pop `.0` values and push them as an array. + /// + /// Only for a literal whose elements are not all constant — one that is + /// gets folded into the pool by rhai's own optimizer before this sees it. + MakeArray(u16), + + /// Build a map from a template and `n` key/value pairs above it. + /// + /// The stack holds `[template, k0, v0, .., k(n-1), v(n-1)]`. The template + /// is a constant map that already carries every key the literal mentions, + /// with the computed ones holding a placeholder — that is rhai's own shape + /// (`ast/expr.rs:283`), and it is why an entirely constant map never + /// reaches here: the optimizer has already folded it into the template + /// alone. + /// + /// Keys ride on the operand stack as string constants rather than in a + /// pool of their own. They are constants either way, and the constant pool + /// already deduplicates them across the program. + MakeMap(u16), + + /// Measure the value on top of the stack into the array literal being + /// built, and raise `ErrorDataTooLarge` if the running total is over. + /// + /// The operand is the element's index within its literal: zero starts a + /// fresh total, and [`Op::MakeArray`] discards it. That is what keeps + /// `[a, [b, c], d]` straight — the inner literal's total is pushed and + /// popped inside the outer one's. + /// + /// A separate instruction rather than work inside `MakeArray` because rhai + /// blames the *element* that tipped the total over + /// (`eval/expr.rs:328`), and one instruction has one position-table entry. + /// Putting it here rather than in a pool beside the element count is what + /// keeps those positions strippable, which matters more for a literal than + /// for a chain: an array can have any number of elements. + CheckSize { + /// The element's index within its literal + index: u16, + /// Whether the element counts towards the map limit rather than the + /// array one. Rhai adds one to a different member of the triple for + /// each (`eval/expr.rs:323` against `:354`), so the same running total + /// cannot serve both. + map: bool, + }, + + /// Walk `a.b[i].c`, indexing the chain pool. + /// + /// One instruction for the whole chain rather than one per step, because + /// the walk holds a `&mut` into the container at every level and a borrow + /// cannot survive a trip round the dispatch loop. Index values and method + /// arguments were pushed before it, in step order. + /// + /// Pushes the value for a read, or unit for an assignment. + Chain(u32), + + /// Truncate the scope back to `.0` locals, dropping everything a block + /// declared. The compile-time slot model unwinds in step. + UnwindTo(u16), + + /// Count one operation against `max_operations`, and give `on_progress` a + /// chance to terminate. + /// + /// Emitted on loop back-edges. Rhai ticks per AST node, so counts differ; + /// what this preserves is that a limit is enforced and an interrupt is + /// honoured, which is what stops `loop {}` from being unkillable. + /// + /// Its table entry is read on every iteration rather than only on failure, + /// which is why the in-memory position table is dense. + Tick, + + /// Record the current scope length as the depth an error escaping this + /// chunk unwinds to. + /// + /// Rhai rewinds a nested block whether it is left normally or by a throw, + /// and never rewinds the top level of a chunk (`eval/stmt.rs`, and + /// `eval_global_statements` passing `rewind_scope = false`). The normal + /// path is [`Op::UnwindTo`], which an escaping error jumps straight past — + /// so the frame needs a floor to fall back to, and the last top-level + /// statement boundary is exactly it. + /// + /// Emitted once before each top-level statement of a chunk that runs in the + /// caller's scope, so it costs nothing per iteration and nothing at all to + /// a function body, whose scope is discarded whole. + Checkpoint, + + /// Evaluate residual AST fragment `residual` through rhai's walker, + /// pushing its value. + /// + /// `rewind_scope` reaches `eval_stmt_block` when the fragment is a block, + /// and decides whether locals it declares survive. Statement fragments + /// rewind, so they cannot disturb the scope shape slots were resolved + /// against. A whole-program fragment does not, because rhai does not + /// rewind top-level statements and callers can see what they declared. + EvalAst { + /// Index into the residual pool + residual: u32, + /// Whether locals the fragment declares are discarded afterwards. + rewind_scope: bool, + }, + + /// Arm a handler covering the instructions up to the matching + /// [`Op::PopHandler`], catching to `target`. + /// + /// Records where the operand stack, the scope and the iterator stack were + /// when it was armed, because an error can be raised at any depth of all + /// three and the catch block has to start where the `try` did. + /// + /// Only errors rhai considers catchable are caught: `return`, `break`, + /// `continue` and `exit` unwind as errors too and must pass straight + /// through (`eval/stmt.rs:806`). + /// + /// `catch_var` names the variable the error is bound to. Its table entry + /// is that variable's position, which is what rhai reports + /// `ErrorTooManyVariables` against. + PushHandler { + /// Where to jump to when an error is caught. + target: u32, + /// The name the error is bound to; absent for a bare `catch`. + catch_var: Option, + }, + + /// Disarm the innermost handler. + /// + /// Emitted twice per `try`: once where the body ends normally, and once + /// where the catch block does — the second ends the region in which a + /// bare `throw;` means "re-raise the original". + PopHandler, + + /// Pop an iterable and start iterating it. + /// + /// The iterator goes on a stack of the VM's own rather than the operand + /// stack, because it is not a `Dynamic`. Rhai's iterator functions take + /// the iterable **by value** and hand back something that cannot be + /// re-created, so it is made once here and lives until the loop ends. + /// + /// Its table entry is the iterable's *start* position, which is what + /// `ErrorFor` is reported against (`eval/stmt.rs:703`) — a different + /// position from the one [`Op::IterNext`] uses. + IterInit, + + /// Advance the current iterator: push the next item and fall through, or + /// drop the iterator and jump to `exit`. + /// + /// The only instruction whose two edges leave different amounts on the + /// operand stack, which is why the verifier gives it explicit successors. + /// + /// Its table entry is the iterable's position — `position`, not + /// `start_position` — because that is what a fallible iterator's error is + /// filled in with (`eval/stmt.rs:749`). + IterNext { + /// Where to jump to once the iterator is exhausted. + exit: u32, + /// `for (x, i) in seq`: the count is pushed under the item, so the two + /// `StoreShared`s that follow pop them in declaration order. + indexed: bool, + }, + + /// Discard the current iterator. + /// + /// Emitted where a `break` leaves a loop, since the jump skips the + /// [`Op::IterNext`] that would have dropped it on exhaustion. Leaving a + /// frame drops whatever it left behind without this. + IterDrop, + + /// Pop a value and write it into local slot `.0`, through a shared cell + /// rather than over it. + /// + /// Distinct from [`Op::StoreLocal`] only in intent: the `for` loop + /// variable is written once per iteration and a closure in the body may + /// have shared it, in which case rhai writes into the cell and every + /// closure made in the loop sees the last value (`eval/stmt.rs:752`). + StoreShared(u16), + + /// Pop a value and raise it as a `throw`. + /// + /// Always fails, with `ErrorRuntime` carrying the value — rhai wraps + /// nothing and converts nothing, so any type can be thrown. Its table + /// entry is the `throw` keyword's own position, not the expression's + /// (`eval/stmt.rs:877`). + Throw, + + /// End the chunk, yielding the top of the operand stack, or unit if empty. + Return, +} diff --git a/src/grain/bytecode/positions.rs b/src/grain/bytecode/positions.rs new file mode 100644 index 000000000..6a3c4e533 --- /dev/null +++ b/src/grain/bytecode/positions.rs @@ -0,0 +1,247 @@ +use crate::grain::pos::Site; +use crate::Position; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +/// Where each instruction came from, or nothing. +/// +/// Instructions carry no position of their own. Diagnostics are the one part of +/// an artifact that is never read unless something has already failed, so they +/// are the part worth being able to leave behind — and an instruction that is +/// pure payload is one that can be executed straight out of a borrowed byte +/// slice, which is the larger reason. +/// +/// Moving them out is close to free on bytes: they were only ever stored on the +/// instructions that had one. What it buys is that they can now be removed. +/// `tests/format.rs` measures how much that removes. +/// +/// In memory this is dense, because the lookup is not always cold. A loop +/// back-edge passes a position to `track_operation` on every iteration, and the +/// built-in operator path builds a `NativeCallContext` around one — both on the +/// hot path, both needing it only if something goes wrong. Indexing an array is +/// what makes that free. The compact delta form in [`pos`](crate::grain::pos) +/// is the wire form, expanded once at load. +/// +/// [`Positions::Stripped`] is not a degraded mode to apologise for: it is what +/// a device runs. Errors come back carrying an instruction address instead of a +/// position, and the host that kept the table resolves it. +#[derive(Debug, Clone, Default)] +pub enum Positions { + /// The table was never written, or was stripped before shipping. + #[default] + Stripped, + /// One entry per instruction, so a lookup is an index. + Dense(Box<[Position]>), +} + +impl Positions { + /// Build from one position per instruction, dropping the table entirely if + /// none of them say anything. + pub(crate) fn dense(positions: Vec) -> Self { + if positions.iter().all(|pos| pos.is_none()) { + return Self::Stripped; + } + Self::Dense(positions.into_boxed_slice()) + } + + /// The position recorded for `pc`, or `Position::NONE`. + /// + /// Out of range reads as no position rather than panicking: this runs while + /// an error is being reported, and losing the position must not replace the + /// error being reported. + #[must_use] + pub fn get(&self, pc: usize) -> Position { + match self { + Self::Stripped => Position::NONE, + Self::Dense(positions) => positions.get(pc).copied().unwrap_or(Position::NONE), + } + } + + /// Whether the position table is stripped + #[must_use] + pub fn is_stripped(&self) -> bool { + matches!(self, Self::Stripped) + } + + /// Encode as the compact table [`pos::resolve`](crate::grain::pos::resolve) + /// reads. + /// + /// Instructions with no position are skipped, which is most of them. + #[must_use] + pub fn to_table(&self) -> Vec { + let Self::Dense(positions) = self else { + return crate::grain::pos::encode(core::iter::empty()); + }; + + crate::grain::pos::encode(positions.iter().enumerate().filter_map(|(pc, pos)| { + let line = pos.line()?; + Some(( + pc as u32, + Site { + line: line as u32, + column: pos.position().unwrap_or(0) as u32, + }, + )) + })) + } + + /// Expand a compact table back to one entry per instruction. + /// + /// `instructions` bounds the result, so a table naming an address past the + /// end of the chunk is refused rather than silently widening the array. + /// + /// # Errors + /// + /// Whatever [`pos::check`](crate::grain::pos::check) found, or [`TableError::PastTheEnd`] + /// for an address that does not name an instruction. + pub fn from_table(table: &[u8], instructions: usize) -> Result { + crate::grain::pos::check(table).map_err(TableError::Malformed)?; + + let count = crate::grain::pos::count(table).map_err(TableError::Malformed)?; + if count == 0 { + return Ok(Self::Stripped); + } + + // `check` proved the table sound, so a miss here is an address that + // does not name an instruction in *this* chunk, which the count + // comparison below catches. + let positions: Vec = (0..instructions) + .map(|pc| { + crate::grain::pos::resolve(table, pc as u32) + .map_or(Position::NONE, site_to_position) + }) + .collect(); + + let found = positions.iter().filter(|pos| !pos.is_none()).count(); + if found != count as usize { + return Err(TableError::PastTheEnd { + entries: count as usize, + matched: found, + instructions, + }); + } + + Ok(Self::dense(positions)) + } +} + +/// Why a position table could not be attached to a chunk. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TableError { + /// The table is not well formed. + Malformed(crate::grain::pos::Error), + /// The table names addresses this chunk does not have, so it belongs to a + /// different program. + PastTheEnd { + /// How many entries the table holds + entries: usize, + /// How many of them landed on an instruction + matched: usize, + /// How many instructions the chunk has + instructions: usize, + }, +} + +impl core::fmt::Display for TableError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Malformed(err) => write!(f, "position table is malformed: {err:?}"), + Self::PastTheEnd { + entries, + matched, + instructions, + } => write!( + f, + "position table has {entries} entries but only {matched} name one of this \ + chunk's {instructions} instructions, so it is a different program's" + ), + } + } +} + +/// A [`Site`] is plain numbers, on purpose — turning it into rhai's own type is +/// this side's job. +fn site_to_position(site: Site) -> Position { + let (Ok(line), Ok(column)) = (u16::try_from(site.line), u16::try_from(site.column)) else { + return Position::NONE; + }; + if line == 0 { + return Position::NONE; + } + Position::new(line, column) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> Positions { + Positions::dense(vec![ + Position::NONE, + Position::new(1, 5), + Position::NONE, + Position::new(3, 0), + ]) + } + + #[test] + fn a_table_survives_the_round_trip() { + let table = sample().to_table(); + let back = Positions::from_table(&table, 4).expect("the table is this chunk's"); + + for pc in 0..4 { + assert_eq!(back.get(pc), sample().get(pc), "at {pc}"); + } + } + + /// Column 0 is the start of a line, not the absence of a position. + #[test] + fn the_start_of_a_line_survives() { + let table = sample().to_table(); + let back = Positions::from_table(&table, 4).unwrap(); + assert_eq!(back.get(3), Position::new(3, 0)); + } + + #[test] + fn stripping_is_what_a_missing_table_reads_as() { + let stripped = Positions::Stripped; + assert!(stripped.is_stripped()); + assert_eq!(stripped.get(0), Position::NONE); + + let empty = Positions::from_table(&stripped.to_table(), 4).unwrap(); + assert!(empty.is_stripped()); + } + + /// A chunk with nothing to say should not carry an array of nothing. + #[test] + fn positions_that_are_all_absent_collapse() { + assert!(Positions::dense(vec![Position::NONE; 8]).is_stripped()); + } + + /// Attaching the wrong program's table would silently misreport every + /// error, which is worse than reporting none. + #[test] + fn a_table_from_another_program_is_refused() { + let table = sample().to_table(); + assert!(matches!( + Positions::from_table(&table, 2), + Err(TableError::PastTheEnd { .. }), + )); + } + + #[test] + fn a_malformed_table_is_refused_rather_than_half_applied() { + let table = sample().to_table(); + assert!(matches!( + Positions::from_table(&table[..table.len() - 1], 4), + Err(TableError::Malformed(..)), + )); + } + + /// Reading past the chunk is how an error report finds out there is no + /// position, and it happens while another error is already in flight. + #[test] + fn an_address_past_the_end_reads_as_no_position() { + assert_eq!(sample().get(9999), Position::NONE); + } +} diff --git a/src/grain/bytecode/strings.rs b/src/grain/bytecode/strings.rs new file mode 100644 index 000000000..01a4bc98e --- /dev/null +++ b/src/grain/bytecode/strings.rs @@ -0,0 +1,231 @@ +use alloc::borrow::Cow; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +/// Every name a program mentions, as one blob and a list of spans. +/// +/// Names are the largest thing a load used to allocate: an `ImmutableString` +/// is a reference-counted box, so a hundred of them is a hundred allocations +/// and several kilobytes. Almost none of them need to be one — a call name, an +/// operator, a getter and a property key are all handed to rhai as `&str`. +/// +/// So the table is a byte blob borrowed straight out of the artifact, and a +/// name is a slice of it. Two allocations for the whole table, and neither +/// scales with how many names there are: the spans, and nothing else. +/// +/// A name that becomes a scope entry is copied rather than borrowed, because +/// `Scope` stores its own. That is a copy into a `SmartString` and not an +/// allocation, so it costs nothing for the short names a `let` or a parameter +/// actually has. +#[derive(Debug, Clone, Default)] +pub struct Strings<'a> { + blob: Cow<'a, [u8]>, + /// Start of each name; the end is the next start, so this is one longer + /// than the number of names. + starts: Vec, +} + +impl<'a> Strings<'a> { + /// Build from names in index order. + #[must_use] + pub fn new>(names: impl IntoIterator) -> Strings<'static> { + let mut blob = Vec::new(); + let mut starts = vec![0u32]; + + for name in names { + blob.extend_from_slice(name.as_ref().as_bytes()); + starts.push(blob.len() as u32); + } + + Strings { + blob: Cow::Owned(blob), + starts, + } + } + + /// Wrap a blob and its spans, as read from an artifact. + /// + /// # Errors + /// + /// [`BadTable`] if the spans do not ascend within the blob, or if any name + /// is not UTF-8 — both of which would otherwise turn into a slice panic + /// while the VM was already running. + pub fn borrowed(blob: &'a [u8], starts: Vec) -> Result { + if starts.is_empty() { + return Err(BadTable::NoTerminator); + } + if starts[0] != 0 { + return Err(BadTable::NoTerminator); + } + + for pair in starts.windows(2) { + let (from, to) = (pair[0] as usize, pair[1] as usize); + if to < from || to > blob.len() { + return Err(BadTable::SpanOutOfRange { from, to }); + } + core::str::from_utf8(&blob[from..to]).map_err(|_| BadTable::NotUtf8 { at: from })?; + } + if *starts.last().expect("checked") as usize != blob.len() { + return Err(BadTable::TrailingBytes); + } + + Ok(Self { + blob: Cow::Borrowed(blob), + starts, + }) + } + + /// How many names the table holds. + #[must_use] + pub fn len(&self) -> usize { + self.starts.len().saturating_sub(1) + } + + /// Whether the table holds no names. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The name at `index`, or `None`. + /// + /// Never allocates: the result points into the artifact. + #[must_use] + pub fn get(&self, index: u32) -> Option<&str> { + let from = *self.starts.get(index as usize)? as usize; + let to = *self.starts.get(index as usize + 1)? as usize; + // Checked once on construction, so this cannot fail. + core::str::from_utf8(self.blob.get(from..to)?).ok() + } + + /// The concatenated names, without their spans. + #[must_use] + pub fn blob(&self) -> &[u8] { + &self.blob + } + + /// The span boundaries, one longer than [`Strings::len`]. + #[must_use] + pub fn starts(&self) -> &[u32] { + &self.starts + } + + /// Iterate the names in index order. + pub fn iter(&self) -> impl Iterator + '_ { + (0..self.len() as u32).filter_map(|index| self.get(index)) + } + + /// Take ownership of the blob, so the table outlives the artifact. + #[must_use] + pub fn into_owned(self) -> Strings<'static> { + Strings { + blob: Cow::Owned(self.blob.into_owned()), + starts: self.starts, + } + } +} + +/// Why a string table could not be used. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BadTable { + /// The spans are empty, or do not start at zero. + NoTerminator, + /// A span runs backwards or past the end of the blob. + SpanOutOfRange { + /// Where the span starts + from: usize, + /// Where it ends + to: usize, + }, + /// A name is not UTF-8. + NotUtf8 { + /// Where the name starts + at: usize, + }, + /// The blob is longer than the last span accounts for. + TrailingBytes, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_come_back_in_order() { + let table = Strings::new(["alpha", "b", "", "gamma"]); + assert_eq!(table.len(), 4); + assert_eq!(table.get(0), Some("alpha")); + assert_eq!(table.get(1), Some("b")); + assert_eq!(table.get(2), Some(""), "an empty name is still a name"); + assert_eq!(table.get(3), Some("gamma")); + assert_eq!(table.get(4), None); + } + + #[test] + fn a_table_survives_being_taken_apart_and_borrowed_back() { + let owned = Strings::new(["one", "two", "three"]); + let blob = owned.blob().to_vec(); + + let borrowed = Strings::borrowed(&blob, owned.starts().to_vec()).expect("sound"); + assert_eq!(borrowed.iter().collect::>(), ["one", "two", "three"]); + } + + #[test] + fn an_empty_table_is_sound() { + let table = Strings::new(Vec::<&str>::new()); + assert!(table.is_empty()); + assert_eq!(table.get(0), None); + + let borrowed = Strings::borrowed(&[], vec![0]).expect("sound"); + assert!(borrowed.is_empty()); + } + + /// The spans come from an artifact, so they are untrusted and must be + /// rejected rather than turned into a slice panic mid-run. + #[test] + fn spans_that_do_not_fit_the_blob_are_refused() { + assert_eq!( + Strings::borrowed(b"abc", vec![]).err(), + Some(BadTable::NoTerminator), + ); + assert_eq!( + Strings::borrowed(b"abc", vec![1, 3]).err(), + Some(BadTable::NoTerminator), + ); + assert!(matches!( + Strings::borrowed(b"abc", vec![0, 99]), + Err(BadTable::SpanOutOfRange { .. }), + )); + assert!(matches!( + Strings::borrowed(b"abc", vec![0, 2, 1]), + Err(BadTable::SpanOutOfRange { .. }), + )); + assert_eq!( + Strings::borrowed(b"abcd", vec![0, 3]).err(), + Some(BadTable::TrailingBytes), + ); + } + + #[test] + fn a_name_that_is_not_utf8_is_refused() { + assert!(matches!( + Strings::borrowed(&[0xff, 0xfe], vec![0, 2]), + Err(BadTable::NotUtf8 { .. }), + )); + } + + /// The property the whole type exists for. + #[test] + fn borrowing_points_into_the_caller_s_buffer() { + let blob = b"alphabeta".to_vec(); + let table = Strings::borrowed(&blob, vec![0, 5, 9]).expect("sound"); + + let name = table.get(0).expect("present"); + assert_eq!(name, "alpha"); + assert!( + name.as_ptr() >= blob.as_ptr() + && (name.as_ptr() as usize) < blob.as_ptr() as usize + blob.len(), + "a borrowed name must point into the blob, not into a copy", + ); + } +} diff --git a/src/grain/bytecode/switch.rs b/src/grain/bytecode/switch.rs new file mode 100644 index 000000000..3e069c01f --- /dev/null +++ b/src/grain/bytecode/switch.rs @@ -0,0 +1,308 @@ +use crate::{ast::RangeCase, Dynamic, INT}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +/// A `switch`, as one dispatch table. +/// +/// Matching a case is *hash* equality, not `==`. That distinction is rhai's +/// and it is visible: `switch 1 { 1.0 => .. }` does not match, because an +/// integer and a float hash differently, while `1 == 1.0` is true. +/// +/// ## Why the hashes travel, and what that costs +/// +/// Rhai's parser keeps only the hash of each case — the value itself is not in +/// the AST (`ast/stmt.rs:336`), so there is nothing to re-hash later. The +/// hashes have to be written out as they are. +/// +/// And by default they do not survive the trip: `get_hasher` falls back to +/// `ahash::AHasher::default()`, and rhai's default features include +/// `ahash/runtime-rng`, so the seed is drawn per process. Rhai gets away with +/// baking hashes into its AST only because it parses and evaluates in one. +/// +/// So an artifact containing a `switch` requires +/// [`rhai::config::hashing::set_hashing_seed`] to have been called with the +/// same seed on both sides. That is not something the format can enforce, but +/// it is something it can *check*: [`probe`] hashes a fixed value, the +/// artifact carries the result, and a loader that computes a different one +/// refuses rather than dispatching every case to the default. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Switch { + /// One entry per distinct case value, in source order. The target is the + /// head of that value's chain of guarded arms. + pub cases: Vec, + /// Checked only when no case matched at all, as rhai does — a case that + /// matched but whose guards all failed goes to the default rather than on + /// to the ranges (`eval/stmt.rs:544`). + /// + /// Disjoint and in ascending order, which rhai's are not: the compiler + /// splits overlapping arms apart so that the first entry containing a + /// value is the only one that can match it. See `compile::cases`. + pub ranges: Vec, + /// Where to go when nothing matched. Always present: an absent `_` arm + /// compiles to a jump past the statement. + pub default: u32, +} + +/// One `value => ...` arm, keyed by rhai's hash of the value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SwitchCase { + /// Rhai's hash of the case value. + pub hash: u64, + /// Where to jump to. + pub target: u32, +} + +/// One `a..b => ...` arm. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SwitchRange { + /// The lower bound + pub from: INT, + /// The upper bound + pub to: INT, + /// Whether the upper bound is included. + pub inclusive: bool, + /// Where to jump to. + pub target: u32, +} + +impl SwitchRange { + /// Whether a subject falls in this range. + /// + /// Delegates to rhai's own `RangeCase` rather than comparing integers, + /// because a range arm matches more than integers: `switch 5.5 { 0..10 => + /// .. }` matches, and under the `decimal` feature so does a `Decimal` + /// (`ast/stmt.rs:254`). Rebuilding the case is two moves and no + /// allocation, and it means there is one definition of what a range arm + /// covers. + #[must_use] + pub fn contains(&self, value: &Dynamic) -> bool { + let case: RangeCase = if self.inclusive { + (self.from..=self.to).into() + } else { + (self.from..self.to).into() + }; + case.contains(value) + } +} + +impl Switch { + /// Where a subject sends control. + /// + /// The order is rhai's (`eval/stmt.rs:517-564`) and each step of it is + /// load-bearing: an unhashable subject reaches neither the cases nor the + /// ranges, and a subject whose hash *did* find a case never reaches the + /// ranges even when that case's guards all decline it — the compiler + /// points such a chain at the default. + #[must_use] + pub fn dispatch(&self, subject: &Dynamic) -> u32 { + // Hashing an unhashable value panics, so this is a guard and not an + // optimization. + if !subject.is_hashable() { + return self.default; + } + + let hash = hash_of(subject); + if let Some(case) = self.cases.iter().find(|case| case.hash == hash) { + return case.target; + } + + // Disjoint, so the first containing entry is the only one. + if let Some(range) = self.ranges.iter().find(|r| r.contains(subject)) { + return range.target; + } + + self.default + } +} + +/// A fixed value hashed with the engine's hasher, so two processes can find +/// out whether their case hashes mean the same thing. +/// +/// Not a checksum of the seed — the seed is not readable as a number the +/// format could compare. This is the observable consequence of it. +#[must_use] +pub fn probe() -> u64 { + hash_of(&Dynamic::from("rhaigrain switch probe")) +} + +fn hash_of(value: &Dynamic) -> u64 { + use core::hash::{Hash, Hasher}; + + let mut hasher = crate::func::get_hasher(); + value.hash(&mut hasher); + hasher.finish() +} + +/// The hash rhai's `switch` would key `value` under. +/// +/// Test-only. The compiler never hashes anything: rhai's parser has already +/// grouped the arms by hash, and the hashes are all it kept. +#[cfg(test)] +fn case_hash(value: &Dynamic) -> Option { + value.is_hashable().then(|| hash_of(value)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A script integer, as a subject or a case. + /// + /// Spelled through `INT` rather than as an `i64` literal because `only_i32` + /// narrows it: a `Dynamic` built from the wider type there is a boxed host + /// value, which has no hash and so matches nothing. + fn int(value: INT) -> Dynamic { + Dynamic::from(value) + } + + /// A host type, which has no hash. + #[derive(Debug, Clone)] + struct Opaque; + + fn table(cases: &[(&Dynamic, u32)], ranges: Vec, default: u32) -> Switch { + Switch { + cases: cases + .iter() + .filter_map(|(value, target)| { + Some(SwitchCase { + hash: case_hash(value)?, + target: *target, + }) + }) + .collect(), + ranges, + default, + } + } + + #[test] + fn a_matching_case_wins() { + let (one, two) = (int(1), int(2)); + let table = table(&[(&one, 10), (&two, 20)], Vec::new(), 99); + + assert_eq!(table.dispatch(&one), 10); + assert_eq!(table.dispatch(&two), 20); + assert_eq!(table.dispatch(&int(3)), 99); + } + + /// The distinction that makes this hashing rather than `==`: rhai does not + /// match an integer against a float case, even though `1 == 1.0`. + #[cfg(not(feature = "no_float"))] + #[test] + fn a_float_does_not_match_an_integer_case() { + let one = int(1); + let table = table(&[(&one, 10)], Vec::new(), 99); + + let float = Dynamic::from(1.0 as crate::FLOAT); + // The subject has to reach the hasher for this to say anything. Built + // from a literal `f64` it would not under `f32_float`; that is a boxed + // host value, and it would land on the default for having no hash at + // all rather than for hashing differently. + assert!(float.is_hashable(), "this test needs a hashable float"); + assert_eq!(table.dispatch(&float), 99); + } + + #[test] + fn strings_and_characters_match_by_value() { + let (text, ch, flag) = ( + Dynamic::from("hello"), + Dynamic::from('x'), + Dynamic::from(true), + ); + let table = table(&[(&text, 10), (&ch, 20), (&flag, 30)], Vec::new(), 99); + + assert_eq!(table.dispatch(&Dynamic::from("hello")), 10); + assert_eq!(table.dispatch(&Dynamic::from("other")), 99); + assert_eq!(table.dispatch(&ch), 20); + assert_eq!(table.dispatch(&flag), 30); + } + + /// A range arm covers the reals between its bounds, not just the integers + /// in them — which is why the check delegates to rhai's own `RangeCase` + /// rather than comparing integers. + #[test] + #[cfg(not(feature = "no_float"))] + fn a_range_catches_a_float_between_its_bounds() { + let table = table( + &[], + vec![SwitchRange { + from: 0, + to: 10, + inclusive: false, + target: 20, + }], + 99, + ); + + // The script float type, not `f64`: under `f32_float` a `Dynamic` + // holding an `f64` is a foreign type and never matches a range arm. + assert_eq!(table.dispatch(&Dynamic::from(5.5 as rhai::FLOAT)), 20); + assert_eq!( + table.dispatch(&Dynamic::from(10.0 as rhai::FLOAT)), + 99, + "exclusive end" + ); + assert_eq!(table.dispatch(&Dynamic::from(-0.5 as rhai::FLOAT)), 99); + } + + /// Ranges are consulted only after the cases miss. + #[test] + fn a_range_catches_what_no_case_did() { + let one = int(1); + let table = table( + &[(&one, 10)], + vec![ + SwitchRange { + from: 5, + to: 8, + inclusive: false, + target: 20, + }, + SwitchRange { + from: 8, + to: 10, + inclusive: true, + target: 30, + }, + ], + 99, + ); + + assert_eq!(table.dispatch(&one), 10, "a case still wins"); + assert_eq!(table.dispatch(&int(5)), 20); + assert_eq!(table.dispatch(&int(7)), 20); + assert_eq!(table.dispatch(&int(8)), 30, "exclusive end"); + assert_eq!(table.dispatch(&int(10)), 30, "inclusive end"); + assert_eq!(table.dispatch(&int(11)), 99); + } + + /// Hashing one would panic, so it must never reach the hasher — and it + /// must still be able to reach the default. + #[test] + fn an_unhashable_subject_falls_through_rather_than_panicking() { + let one = int(1); + let table = table(&[(&one, 10)], Vec::new(), 99); + + // A bare function pointer *is* hashable; only one carrying an + // environment is not. A host type is the reliable case. + let unhashable = Dynamic::from(Opaque); + assert!( + !unhashable.is_hashable(), + "this test needs an unhashable value", + ); + assert_eq!(table.dispatch(&unhashable), 99); + } + + #[test] + fn an_unhashable_case_has_no_hash_to_key_on() { + assert_eq!(case_hash(&Dynamic::from(Opaque)), None); + assert!(case_hash(&int(1)).is_some()); + } + + /// The probe is only worth carrying if it actually depends on the seed. + #[test] + fn the_probe_is_stable_within_a_process() { + assert_eq!(probe(), probe()); + assert_ne!(probe(), 0, "a probe of zero could not be told from absent"); + } +} diff --git a/src/grain/bytecode/verify.rs b/src/grain/bytecode/verify.rs new file mode 100644 index 000000000..6adba7a32 --- /dev/null +++ b/src/grain/bytecode/verify.rs @@ -0,0 +1,1025 @@ +use crate::grain::bytecode::code::{self, tag}; +use crate::grain::bytecode::{Chain, Chunk, Op, Receiver, Root, Step, Switch, Tail}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +/// What the pools hold, so an instruction's indices can be checked against +/// something. +/// +/// Chains and switches come through whole rather than as a count, because both +/// hold things that have to be checked rather than counted: how much operand +/// stack a chain consumes, and where a switch can send control. +#[derive(Debug, Clone, Copy)] +pub struct Pools<'a> { + /// How many constants there are. + pub consts: usize, + /// How many interned names there are. + pub names: usize, + /// How many operator tokens there are. + pub tokens: usize, + /// How many op-assignments there are. + pub assign_ops: usize, + /// How many residual AST fragments there are. + pub residuals: usize, + /// The chain pool. + pub chains: &'a [Chain], + /// The switch pool. + pub switches: &'a [Switch], +} + +/// Why a chunk was rejected. +/// +/// Every variant names something a correct compiler cannot produce, so a +/// failure here is a bug in the compiler or a corrupted artifact — never +/// anything a script can express. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VerifyError { + /// A tag with no instruction behind it, or one whose operands run past the + /// end of the chunk. + Undecodable { + /// Byte offset of the offending tag + at: usize, + }, + /// The last instruction stops short of the end, so the trailing bytes are + /// not instructions. + TrailingBytes { + /// Where the trailing bytes start + at: usize, + /// How long the code is + len: usize, + }, + /// A chunk names a span the code does not have. + ChunkOutOfRange { + /// The chunk's first byte offset + entry: u32, + /// One past its last + end: u32, + /// How long the code is + len: usize, + }, + /// A jump leaves the chunk it is in — including into another chunk, which + /// would run that function's instructions against this frame's locals. + JumpOutOfRange { + /// Byte offset of the jump + at: usize, + /// The byte offset it names + target: u32, + }, + /// A jump lands inside an instruction rather than on one. Decoding from + /// there would read an operand's bytes as a tag. + JumpIntoAnInstruction { + /// Byte offset of the jump + at: usize, + /// The byte offset it names + target: u32, + }, + /// Two paths reach the same instruction with different stack depths, so + /// the depth at that point is not a static property. + DepthConflict { + /// Byte offset of the instruction + at: usize, + /// The depth already recorded for it + expected: usize, + /// The depth the other path arrives with + found: usize, + }, + /// An instruction pops more than is on the stack. + Underflow { + /// Byte offset of the instruction + at: usize, + /// How many values it pops + need: usize, + /// How many are on the stack + have: usize, + }, + /// Execution can run past the last instruction. + FallsOffTheEnd, + /// The chunk claims less stack than it uses. + StackExceedsDeclared { + /// The depth actually reached + needed: usize, + /// The depth the chunk declares + declared: u16, + }, + /// An iterator is dropped where none was made. The compiler pairs these + /// up lexically; an artifact off a wire has to be asked. + IteratorUnderflow { + /// Byte offset of the instruction + at: usize, + }, + /// A handler is disarmed where none was armed. A stale handler is worse + /// than a missing one: the next unrelated error would be caught into an + /// already-exited `catch` block. + HandlerUnderflow { + /// Byte offset of the instruction + at: usize, + }, + /// An index into a pool with nothing behind it. + BadIndex { + /// Byte offset of the instruction + at: usize, + /// Which pool was indexed + what: &'static str, + /// The index it used + index: u32, + }, +} + +/// Check that a chunk is internally consistent before running it. +/// +/// Two passes. The first decodes straight through, recording where each +/// instruction starts; that is what makes it safe to say a jump target is or is +/// not an instruction, which a reachability walk alone cannot — a jump into the +/// middle of an operand would decode the operand's bytes as a tag and look +/// perfectly reasonable. +/// +/// The second is an abstract interpretation over stack depth: walk every +/// reachable instruction, and require that all paths into one agree on how deep +/// the operand stack is. That single property catches the whole class of +/// compiler bugs where one branch of a conditional leaves a value and the other +/// does not — which is otherwise invisible until a program takes the unlucky +/// path. +/// +/// Together they are what makes an artifact safe to execute in place: a chunk +/// that passes cannot underflow the operand stack, jump outside itself, or +/// decode an operand as an instruction. What it does *not* prove is +/// termination — a jump target inside the chunk is well-formed whether or not +/// it closes a loop — which is why [`Op::Tick`] sits on every back edge and why +/// a host running untrusted bytecode still needs `max_operations`. +/// +/// Returns the measured stack high water, which is what the chunk should +/// declare. +pub fn verify(code: &[u8], chunks: &[Chunk], pools: Pools) -> Result, VerifyError> { + // Pass one: where do instructions start? + // + // Over the whole buffer at once, because every chunk shares it and an + // instruction boundary is a property of the bytes, not of who runs them. + let mut starts = vec![false; code.len() + 1]; + let mut at = 0usize; + while at < code.len() { + starts[at] = true; + let width = code::width(code, at).ok_or(VerifyError::Undecodable { at })?; + check_indices(at, code, pools)?; + at += width; + } + if at != code.len() { + return Err(VerifyError::TrailingBytes { + at, + len: code.len(), + }); + } + + chunks + .iter() + .map(|chunk| verify_chunk(code, chunk, &starts, pools)) + .collect() +} + +/// What every path into an instruction has to agree on. +/// +/// The operand stack is the obvious one. The iterator stack is here for the +/// same reason: a `for` loop's iterator lives on a stack of the VM's own, and +/// a chunk that leaves one behind — or drops one it never made — is a chunk +/// whose loops are not the shape the compiler thought. "The compiler balances +/// them" is exactly the sort of claim a verifier for untrusted bytecode exists +/// to check rather than take on trust. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct State { + operands: usize, + iters: usize, + handlers: usize, +} + +/// Walk one chunk's reachable instructions, checking that every path into an +/// instruction agrees on the stack depth. +fn verify_chunk( + code: &[u8], + chunk: &Chunk, + starts: &[bool], + pools: Pools, +) -> Result { + let (entry, end) = (chunk.entry() as usize, chunk.end() as usize); + if end > code.len() || entry > end { + return Err(VerifyError::ChunkOutOfRange { + entry: chunk.entry(), + end: chunk.end(), + len: code.len(), + }); + } + + let mut depth_at: Vec> = vec![None; code.len()]; + let mut worklist = vec![(entry, State::default())]; + let mut high_water = 0usize; + + while let Some((at, state)) = worklist.pop() { + if at >= end { + return Err(VerifyError::FallsOffTheEnd); + } + + // Merge point: either this is the first visit, or every earlier path + // has to have arrived in the same state. + match depth_at[at] { + Some(seen) if seen == state => continue, + Some(seen) => { + return Err(VerifyError::DepthConflict { + at, + expected: seen.operands, + found: state.operands, + }) + } + None => depth_at[at] = Some(state), + } + + let depth = state.operands; + high_water = high_water.max(depth); + + let op = code::decode(code, at).ok_or(VerifyError::Undecodable { at })?; + let (pops, pushes) = effect(&op, pools); + + if depth < pops { + return Err(VerifyError::Underflow { + at, + need: pops, + have: depth, + }); + } + let next_state = State { + operands: depth - pops + pushes, + iters: match op { + Op::IterInit => state.iters + 1, + Op::IterDrop => state + .iters + .checked_sub(1) + .ok_or(VerifyError::IteratorUnderflow { at })?, + _ => state.iters, + }, + handlers: match op { + Op::PushHandler { .. } => state.handlers + 1, + Op::PopHandler => state + .handlers + .checked_sub(1) + .ok_or(VerifyError::HandlerUnderflow { at })?, + _ => state.handlers, + }, + }; + let next_depth = next_state.operands; + high_water = high_water.max(next_depth); + + let width = code::width(code, at).expect("decoded, so it has a width"); + let next = at + width; + + let mut go = |target: u32, state: State| -> Result<(), VerifyError> { + let target = target as usize; + // Within this chunk: a jump into another function's body would run + // its instructions against this frame's locals. + if target < entry || target >= end { + return Err(VerifyError::JumpOutOfRange { + at, + target: target as u32, + }); + } + if !starts[target] { + return Err(VerifyError::JumpIntoAnInstruction { + at, + target: target as u32, + }); + } + worklist.push((target, state)); + Ok(()) + }; + + match op { + // Terminal: nothing follows. A `throw` always fails, so control + // leaves the chunk here as surely as it does at a `Return`. + // + // Neither has to balance the iterator stack: leaving a frame + // truncates it to what the frame started with. + Op::Return | Op::Throw => {} + + Op::Jump(target) => go(target, next_state)?, + + // The catch block is entered on the exception path, so it starts + // where the `try` did: same operand depth, same iterators, and + // inside the handler it will disarm itself. + Op::PushHandler { target, .. } => { + go( + target, + State { + operands: depth, + iters: state.iters, + handlers: next_state.handlers, + }, + )?; + worklist.push((next, next_state)); + } + + Op::JumpIfFalse { target } | Op::JumpIfTrue { target } => { + go(target, next_state)?; + worklist.push((next, next_state)); + } + + // The one instruction whose edges differ in more than where they + // go: falling through carries the item it pushed and still holds + // the iterator, while the exit edge has neither. + Op::IterNext { exit, indexed } => { + go( + exit, + State { + operands: depth, + iters: state + .iters + .checked_sub(1) + .ok_or(VerifyError::IteratorUnderflow { at })?, + handlers: state.handlers, + }, + )?; + worklist.push(( + next, + State { + // The item, and the count under it when there is one. + operands: depth + 1 + usize::from(indexed), + iters: state.iters, + handlers: state.handlers, + }, + )); + } + + // Terminal like `Jump`, with one successor per arm. A table with + // no entry behind it is caught by `check_indices`, which has + // already run over every instruction. + Op::Switch(index) => { + if let Some(table) = pools.switches.get(index as usize) { + for target in table + .cases + .iter() + .map(|case| case.target) + .chain(table.ranges.iter().map(|range| range.target)) + .chain(core::iter::once(table.default)) + { + go(target, next_state)?; + } + } + } + + _ => { + if next >= end { + return Err(VerifyError::FallsOffTheEnd); + } + worklist.push((next, next_state)); + } + } + } + + let Ok(high_water) = u16::try_from(high_water) else { + return Err(VerifyError::StackExceedsDeclared { + needed: high_water, + declared: chunk.max_stack(), + }); + }; + if high_water > chunk.max_stack() { + return Err(VerifyError::StackExceedsDeclared { + needed: high_water as usize, + declared: chunk.max_stack(), + }); + } + + Ok(high_water) +} + +/// How many operands an instruction consumes and produces. +fn effect(op: &Op, pools: Pools) -> (usize, usize) { + match op { + // A chain eats the indices and arguments its steps named, plus a root + // that is not a slot, plus the value being assigned, and leaves one + // behind. An index with no chain behind it reads as consuming nothing; + // `check_indices` is what rejects it. + Op::Chain(index) => match pools.chains.get(*index as usize) { + Some(chain) => (chain.consumes(), 1), + None => (0, 1), + }, + + Op::Const(..) + | Op::Unit + | Op::Bool(..) + | Op::LoadLocal(..) + | Op::LoadNamed(..) + | Op::LoadShared(..) + | Op::LoadSharedNamed(..) + | Op::MakeClosure(..) + | Op::LoadThis + | Op::LoadThisShared + | Op::EvalAst { .. } => (0, 1), + + // A boundness check, which either raises or does nothing. + Op::RequireThis => (0, 0), + + // Sharing is a change to the scope, not to the operand stack. + Op::Share(..) | Op::ShareNamed(..) => (0, 0), + + Op::StoreLocal(..) | Op::DeclareLocal { .. } | Op::Pop => (1, 0), + + // Pops the value, leaves nothing: the statement's unit value is a + // separate `Op::Unit`. + Op::AssignLocal { .. } | Op::AssignNamed { .. } | Op::AssignThis { .. } => (1, 0), + + Op::JumpIfFalse { .. } | Op::JumpIfTrue { .. } | Op::Switch(..) => (1, 0), + + Op::Jump(..) + | Op::UnwindTo(..) + | Op::Tick + | Op::Checkpoint + | Op::PushHandler { .. } + | Op::PopHandler => (0, 0), + + // Arguments in, result out. + Op::Call { argc, .. } => (*argc as usize, 1), + + // A named receiver's value is argument zero like any other, and so is + // `this` — which is pushed first rather than last, but the depth is the + // same either way. A local's is not on the stack at all. An `argc` of + // zero names no receiver and is rejected when it runs. + Op::CallRef { argc, receiver, .. } => match receiver { + Receiver::Local(..) => ((*argc as usize).saturating_sub(1), 1), + Receiver::Named(..) | Receiver::This => (*argc as usize, 1), + }, + + // Reorders what is already there, so the depth is unchanged — but it + // has to reach every one of them, and saying so is what stops a + // hand-made artifact reaching under the frame. + Op::Rotate(under) => (*under as usize + 1, *under as usize + 1), + + Op::MakeArray(len) => (*len as usize, 1), + + // A key and a value per entry, and the template underneath them. + Op::MakeMap(len) => (2 * *len as usize + 1, 1), + + // Measures the element it is standing on without taking it: the + // literal is still being built and every element it has so far is + // still on the stack. + Op::CheckSize { .. } => (1, 1), + + // The buffer is an ordinary operand: started, appended to, then + // replaced by the string it built. + // A name in, a pointer out. + Op::MakeFnPtr | Op::IsShared => (1, 1), + // The arguments and the pointer itself, leaving one of each. + Op::Curry(argc) => (*argc as usize + 1, 1), + Op::CallFnPtr { argc, .. } => (*argc as usize + 1, 1), + + Op::InterpolateStart => (0, 1), + Op::InterpolateAppend => (1, 0), + Op::InterpolateEnd => (1, 1), + + // Pops the thrown value; nothing follows, so what it leaves is moot. + Op::Throw | Op::StoreShared(..) => (1, 0), + + // The iterable goes onto the iterator stack, not back onto this one. + Op::IterInit => (1, 0), + // Its two edges disagree, so the successor match does the work. + Op::IterNext { .. } | Op::IterDrop => (0, 0), + + // Consumes whatever is left, so depth afterwards is not meaningful. + Op::Return => (0, 0), + } +} + +/// Check that every pool reference resolves. +/// +/// The VM treats these as assertions, and an artifact is the one place they can +/// be wrong without a compiler bug. Reads the operands off the bytes rather +/// than off a decoded `Op`, so it runs in the same pass that measures widths. +fn check_indices(at: usize, code: &[u8], pools: Pools) -> Result<(), VerifyError> { + let index = |offset: usize| code::u16_at(code, at + offset).map_or(0, u32::from); + let bounded = |index: u32, what: &'static str, len: usize| { + if index as usize >= len { + Err(VerifyError::BadIndex { at, what, index }) + } else { + Ok(()) + } + }; + + match code[at] { + tag::CONST => bounded(index(1), "constant", pools.consts), + tag::DECLARE_LOCAL | tag::DECLARE_CONST => bounded(index(1), "name", pools.names), + tag::CALL | tag::CALL_LOCAL_REF | tag::CALL_THIS_REF => { + bounded(index(1), "name", pools.names) + } + // The function's, then the receiver variable's. The slot a local + // receiver names is not a pool index and is checked against the scope + // when it runs, as every other slot is. + tag::CALL_NAMED_REF => { + bounded(index(1), "name", pools.names)?; + bounded(index(4), "name", pools.names) + } + tag::CALL_OP => { + bounded(index(1), "name", pools.names)?; + bounded(index(4), "operator", pools.tokens) + } + tag::ASSIGN_LOCAL => bounded(index(3), "name", pools.names), + tag::LOAD_NAMED + | tag::LOAD_SHARED_NAMED + | tag::ASSIGN_NAMED + | tag::SHARE_NAMED + | tag::MAKE_CLOSURE => bounded(index(1), "name", pools.names), + tag::ASSIGN_NAMED_OP => { + bounded(index(1), "name", pools.names)?; + bounded(index(3), "op-assignment", pools.assign_ops) + } + tag::ASSIGN_LOCAL_OP => { + bounded(index(3), "name", pools.names)?; + bounded(index(5), "op-assignment", pools.assign_ops) + } + // `this` needs no name, so the operator is the whole of it — and + // omitting this would hand `program.assign_op` an unchecked index out + // of a corrupt artifact. + tag::ASSIGN_THIS_OP => bounded(index(1), "op-assignment", pools.assign_ops), + // The receiver's name, which the write-back resolves the scope entry + // by. A local's slot is not a pool index and is checked against the + // scope when it runs, as every other slot is. + tag::CALL_FN_PTR_ON_NAMED => bounded(index(2), "name", pools.names), + tag::EVAL_AST | tag::EVAL_AST_KEEP => bounded(index(1), "fragment", pools.residuals), + tag::CHAIN => { + bounded(index(1), "chain", pools.chains.len())?; + check_chain_indices(at, &pools.chains[index(1) as usize], pools) + } + tag::SWITCH => bounded(index(1), "switch", pools.switches.len()), + _ => Ok(()), + } +} + +/// Check the pool references *inside* a chain record. +/// +/// A chain is one instruction over an unbounded record, so nearly all of what +/// it names lives in the pool rather than in the code. Bounding only the +/// record's own index would leave most of the instruction unverified. +fn check_chain_indices(at: usize, chain: &Chain, pools: Pools) -> Result<(), VerifyError> { + let bounded = |index: u32, what: &'static str, len: usize| { + if index as usize >= len { + Err(VerifyError::BadIndex { at, what, index }) + } else { + Ok(()) + } + }; + + match chain.root { + Root::Local { name, .. } | Root::Named { name, .. } => { + bounded(name, "name", pools.names)?; + } + // Neither names anything in a pool: a temporary has no name at all, and + // `this` is a register rather than an entry. + Root::This { .. } | Root::Temporary => {} + } + + for step in &chain.steps { + match step { + // Its operands are stack offsets and its positions are its own. + Step::Index { .. } => {} + Step::Property { + name, + getter, + setter, + .. + } => { + bounded(*name, "name", pools.names)?; + bounded(*getter, "name", pools.names)?; + bounded(*setter, "name", pools.names)?; + } + Step::Method { name, .. } => bounded(*name, "name", pools.names)?, + } + } + + match chain.tail { + Tail::Assign { op: Some(op) } => bounded(op, "op-assignment", pools.assign_ops), + Tail::Assign { op: None } | Tail::Read => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::grain::bytecode::assemble; + + fn pools() -> Pools<'static> { + Pools { + consts: 0, + names: 0, + tokens: 0, + assign_ops: 0, + residuals: 0, + chains: &[], + switches: &[], + } + } + + /// Assemble one chunk spanning the whole buffer, and check it. + fn check(ops: Vec) -> Result, VerifyError> { + let (code, _) = assemble(&ops).expect("the test ops must assemble"); + let chunk = Chunk::new(0, code.len() as u32, 8); + verify(&code, &[chunk], pools()) + } + + /// The same, for bytes `assemble` would refuse to produce — which is what + /// a corrupt artifact hands the loader. + fn check_bytes(code: Vec, max_stack: u16) -> Result, VerifyError> { + let chunk = Chunk::new(0, code.len() as u32, max_stack); + verify(&code, &[chunk], pools()) + } + + #[test] + fn accepts_a_well_formed_chunk() { + assert_eq!(check(vec![Op::Unit, Op::Return]), Ok(vec![1])); + } + + /// `this` is a register, so reading it costs a push and nothing else, and + /// assigning to it consumes one without leaving anything behind. + #[test] + fn the_this_register_is_reached_without_touching_the_scope() { + assert_eq!(check(vec![Op::LoadThis, Op::Return]), Ok(vec![1])); + assert_eq!(check(vec![Op::LoadThisShared, Op::Return]), Ok(vec![1])); + assert_eq!( + check(vec![Op::RequireThis, Op::Unit, Op::Return]), + Ok(vec![1]) + ); + assert_eq!( + check(vec![ + Op::RequireThis, + Op::Unit, + Op::AssignThis { op: None }, + Op::Unit, + Op::Return + ]), + Ok(vec![1]) + ); + } + + /// The operator is the whole of `ASSIGN_THIS_OP`'s payload, so an artifact + /// naming one the pool does not have has to be refused here — nothing + /// downstream re-checks it. + #[test] + fn rejects_an_op_assignment_to_this_that_the_pool_does_not_have() { + let ops = vec![ + Op::Unit, + Op::AssignThis { op: Some(0) }, + Op::Unit, + Op::Return, + ]; + assert_eq!( + check(ops), + Err(VerifyError::BadIndex { + at: 1, + what: "op-assignment", + index: 0, + }) + ); + } + + /// A chain is one instruction over an unbounded record, so almost all of it + /// is in the pool rather than in the code. Checking only the record's own + /// index would leave the rest of the instruction unverified. + #[test] + fn rejects_a_chain_that_names_something_the_pools_do_not_have() { + let chain = |root, steps, tail| Chain { + root, + steps, + tail, + operands: 0, + }; + let property = |name| Step::Property { + name, + getter: 0, + setter: 0, + pos: rhai::Position::NONE, + }; + + let past_the_end = [ + chain( + Root::Named { + name: 3, + pos: rhai::Position::NONE, + }, + vec![], + Tail::Read, + ), + chain(Root::Local { slot: 0, name: 3 }, vec![], Tail::Read), + chain(Root::Temporary, vec![property(3)], Tail::Read), + chain( + Root::Temporary, + vec![Step::Method { + name: 3, + argc: 0, + operand: 0, + pos: rhai::Position::NONE, + }], + Tail::Read, + ), + ]; + + for chain in past_the_end { + let temporary = chain.roots_on_stack(); + let mut ops = vec![Op::Chain(0), Op::Return]; + if temporary { + ops.insert(0, Op::Unit); + } + let (code, _) = assemble(&ops).expect("must assemble"); + let chunk = Chunk::new(0, code.len() as u32, 8); + let pools = Pools { + names: 1, + chains: core::slice::from_ref(&chain), + ..pools() + }; + assert!( + matches!( + verify(&code, &[chunk], pools), + Err(VerifyError::BadIndex { + what: "name", + index: 3, + .. + }), + ), + "{chain:?} names name 3 of 1 and must be refused", + ); + } + + // And the op-assignment a tail can carry. + let assigning = chain( + Root::Local { slot: 0, name: 0 }, + vec![], + Tail::Assign { op: Some(2) }, + ); + let (code, _) = assemble(&[Op::Unit, Op::Chain(0), Op::Return]).expect("must assemble"); + let chunk = Chunk::new(0, code.len() as u32, 8); + assert!(matches!( + verify( + &code, + &[chunk], + Pools { + names: 1, + chains: core::slice::from_ref(&assigning), + ..pools() + }, + ), + Err(VerifyError::BadIndex { + what: "op-assignment", + index: 2, + .. + }), + )); + } + + /// Chunks share one buffer, so a jump from one into another would run the + /// callee's instructions against the caller's locals. + #[test] + fn rejects_a_jump_from_one_chunk_into_another() { + // Two chunks: `Unit; Return` twice. The first jumps into the second. + let (mut code, _) = assemble(&[Op::Unit, Op::Return]).unwrap(); + let boundary = code.len() as u32; + code.push(tag::JUMP); + code.extend_from_slice(&0u32.to_le_bytes()); // back into chunk one + code.push(tag::RETURN); + + let chunks = [ + Chunk::new(0, boundary, 8), + Chunk::new(boundary, code.len() as u32, 8), + ]; + assert!(matches!( + verify(&code, &chunks, pools()), + Err(VerifyError::JumpOutOfRange { .. }), + )); + } + + /// The reason the high water is returned rather than merely checked: the + /// compiler's estimate is one slot per instruction, and the VM reserves + /// from it. + #[test] + fn the_high_water_is_what_the_chunk_uses_not_what_it_declares() { + assert_eq!( + check(vec![ + Op::Unit, + Op::Unit, + Op::Pop, + Op::Pop, + Op::Unit, + Op::Return + ]), + Ok(vec![2]), + ); + } + + /// The property the verifier exists for: one branch leaves a value, the + /// other does not, and nothing notices until a program takes the wrong + /// path at runtime. + #[test] + fn rejects_branches_that_disagree_on_depth() { + let ops = vec![ + Op::Bool(true), + Op::JumpIfFalse { target: 3 }, + Op::Unit, // the taken path pushes + Op::Return, + ]; + + assert!( + matches!(check(ops.clone()), Err(VerifyError::DepthConflict { .. })), + "a branch imbalance must be rejected, got {:?}", + check(ops), + ); + } + + #[test] + fn rejects_a_jump_off_the_end() { + // Assembled by hand: `assemble` refuses an index it cannot resolve, so + // an out-of-range *address* can only come from a corrupt artifact. + let mut code = vec![tag::JUMP]; + code.extend_from_slice(&99u32.to_le_bytes()); + code.push(tag::RETURN); + + assert!(matches!( + check_bytes(code, 8), + Err(VerifyError::JumpOutOfRange { .. }), + )); + } + + /// A jump into an operand would read that operand's bytes as a tag, which + /// is how a byte-addressed chunk goes wrong in a way an index-addressed one + /// could not. + #[test] + fn rejects_a_jump_into_the_middle_of_an_instruction() { + let mut code = vec![tag::JUMP]; + code.extend_from_slice(&3u32.to_le_bytes()); // lands inside itself + code.push(tag::RETURN); + + assert_eq!( + check_bytes(code, 8), + Err(VerifyError::JumpIntoAnInstruction { at: 0, target: 3 }), + ); + } + + /// A switch is a jump with many targets, and every one of them needs the + /// proof an ordinary jump gets — otherwise the one arm nobody tested is + /// the one that decodes an operand as an opcode. + #[test] + fn every_arm_of_a_switch_is_checked() { + let ops = vec![ + Op::Unit, + Op::Switch(0), + Op::Unit, // index 2: the case arm + Op::Return, + Op::Unit, // index 4: the default + Op::Return, + ]; + let (code, offsets) = assemble(&ops).expect("must assemble"); + let chunk = Chunk::new(0, code.len() as u32, 8); + + let table = |case: u32, default: u32| Switch { + cases: vec![crate::grain::bytecode::SwitchCase { + hash: 7, + target: case, + }], + ranges: Vec::new(), + default, + }; + + let good = [table(offsets[2], offsets[4])]; + assert_eq!( + verify( + &code, + &[chunk], + Pools { + switches: &good, + ..pools() + } + ), + Ok(vec![1]), + ); + + // One byte into the `Switch` instruction's own operand. + let mid = [table(offsets[1] + 1, offsets[4])]; + assert!( + matches!( + verify( + &code, + &[chunk], + Pools { + switches: &mid, + ..pools() + } + ), + Err(VerifyError::JumpIntoAnInstruction { .. }), + ), + "a case arm landing mid-instruction must be refused", + ); + + // The default is a target like any other, and the easiest to forget. + let outside = [table(offsets[2], 9999)]; + assert!( + matches!( + verify( + &code, + &[chunk], + Pools { + switches: &outside, + ..pools() + } + ), + Err(VerifyError::JumpOutOfRange { .. }), + ), + "a default outside the chunk must be refused", + ); + } + + #[test] + fn rejects_popping_an_empty_stack() { + assert!(matches!( + check(vec![Op::Pop, Op::Return]), + Err(VerifyError::Underflow { .. }), + )); + } + + /// A rotate reaches under the operands above it, and one frame's operands + /// sit on the same stack as its caller's. Nothing at run time knows where + /// the frame started, so the depth it needs is checked here or nowhere. + #[test] + fn rejects_a_rotate_that_reaches_below_the_frame() { + assert!(matches!( + check(vec![Op::Unit, Op::Unit, Op::Rotate(2), Op::Return]), + Err(VerifyError::Underflow { + need: 3, + have: 2, + .. + }), + )); + assert_eq!( + check(vec![ + Op::Unit, + Op::Unit, + Op::Unit, + Op::Rotate(2), + Op::Return + ]), + Ok(vec![3]), + "with the third operand there it is in range, and nothing moves", + ); + } + + #[test] + fn rejects_running_past_the_last_instruction() { + assert_eq!(check(vec![Op::Unit]), Err(VerifyError::FallsOffTheEnd)); + } + + #[test] + fn rejects_an_index_with_nothing_behind_it() { + let (code, _) = assemble(&[Op::Const(7), Op::Return]).unwrap(); + let chunk = Chunk::new(0, code.len() as u32, 8); + assert!(matches!( + verify( + &code, + &[chunk], + Pools { + consts: 1, + ..pools() + } + ), + Err(VerifyError::BadIndex { + what: "constant", + .. + }), + )); + } + + #[test] + fn rejects_a_chunk_that_outgrows_its_declared_stack() { + let (code, _) = assemble(&[Op::Unit, Op::Unit, Op::Unit, Op::Return]).unwrap(); + assert!(matches!( + check_bytes(code, 2), + Err(VerifyError::StackExceedsDeclared { .. }), + )); + } + + /// Bytes that are not instructions must be named as such rather than + /// executed. + #[test] + fn rejects_a_tag_it_does_not_know() { + assert_eq!( + check_bytes(vec![0xff], 8), + Err(VerifyError::Undecodable { at: 0 }), + ); + } + + #[test] + fn rejects_an_instruction_whose_operands_are_cut_off() { + assert_eq!( + check_bytes(vec![tag::CONST, 0], 8), + Err(VerifyError::Undecodable { at: 0 }), + ); + } + + /// A chunk naming a span the code does not have is a corrupt artifact, not + /// a compiler bug — and must not index out of bounds. + #[test] + fn rejects_a_chunk_that_names_code_it_does_not_have() { + let (code, _) = assemble(&[Op::Unit, Op::Return]).unwrap(); + assert!(matches!( + verify(&code, &[Chunk::new(0, 9999, 8)], pools()), + Err(VerifyError::ChunkOutOfRange { .. }), + )); + } +} diff --git a/src/grain/compile/cases.rs b/src/grain/compile/cases.rs new file mode 100644 index 000000000..22732a718 --- /dev/null +++ b/src/grain/compile/cases.rs @@ -0,0 +1,302 @@ +//! Turning rhai's `switch` range arms into something that can be looked up. +//! +//! Rhai scans its range arms in order and takes the first one that both +//! contains the subject *and* whose guard passes — so when two arms overlap, +//! which arm runs is not a property of the subject alone. A dispatch table has +//! one answer per subject, so the overlap has to go somewhere, and the only +//! place it can go without keeping the subject alive across the guards is +//! here. +//! +//! [`split`] cuts the arms into disjoint intervals. Each interval carries the +//! arms covering it, in source order, and the compiler emits those as a chain +//! of guards ending at the default — exactly what it already does for the arms +//! sharing a case value. + +use crate::{ast::RangeCase, INT}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +use crate::grain::bytecode::SwitchRange; + +/// A range arm's interval, as bounds rather than as a `Range`. +fn bounds(range: &RangeCase) -> (INT, INT, bool) { + match range { + RangeCase::ExclusiveInt(r, ..) => (r.start, r.end, false), + RangeCase::InclusiveInt(r, ..) => (*r.start(), *r.end(), true), + } +} + +/// A piece of the number line that every arm either covers whole or not at +/// all. +/// +/// Two kinds because the bounds are integers but the values are not: between +/// two adjacent bounds there is nothing to enumerate and yet `5.5` is in +/// there, so the gaps are intervals in their own right rather than something +/// the endpoints cover. +enum Atom { + /// A single integer. + Point(INT), + /// Everything strictly between two adjacent bounds. + Between(INT, INT), +} + +fn covers(range: &RangeCase, atom: &Atom) -> bool { + match atom { + Atom::Point(point) => range.contains_int(*point), + // Every endpoint is a bound, so a range that reaches into a gap + // between two adjacent bounds spans the whole of it. + Atom::Between(low, high) => { + let (start, end, ..) = bounds(range); + start <= *low && end >= *high + } + } +} + +/// One interval and the arms that cover it, being built. +struct Run { + from: INT, + to: INT, + inclusive: bool, + blocks: Vec, +} + +/// Split range arms into disjoint intervals, each carrying the arms that cover +/// it in source order. +/// +/// Targets come back as zero: which instruction an interval sends control to +/// is the caller's to fill in, once it has emitted the chain. +/// +/// ## Why the lower bounds can overlap after all +/// +/// An interval that starts in a gap — everything above `10` but not `10` +/// itself — has no exact form here, and comes back as `10..hi`, which does +/// include `10`. That is sound because entries are scanned in order and `10` +/// is always claimed by an earlier one: any arm reaching into the gap above a +/// bound also covers the bound, so the point is never an interval nobody +/// emitted. +pub(crate) fn split(ranges: &[RangeCase]) -> Vec<(SwitchRange, Vec)> { + let mut points: Vec = Vec::with_capacity(ranges.len() * 2); + for range in ranges { + let (start, end, ..) = bounds(range); + points.push(start); + points.push(end); + } + points.sort_unstable(); + points.dedup(); + + let mut atoms = Vec::with_capacity(points.len() * 2); + for (index, point) in points.iter().enumerate() { + atoms.push(Atom::Point(*point)); + if let Some(next) = points.get(index + 1) { + atoms.push(Atom::Between(*point, *next)); + } + } + + let mut out: Vec<(SwitchRange, Vec)> = Vec::new(); + let mut run: Option = None; + + let finish = |run: Run| { + ( + SwitchRange { + from: run.from, + to: run.to, + inclusive: run.inclusive, + target: 0, + }, + run.blocks, + ) + }; + + for atom in &atoms { + let blocks: Vec = ranges + .iter() + .filter(|range| covers(range, atom)) + .map(RangeCase::index) + .collect(); + + let (from, to, inclusive) = match atom { + Atom::Point(point) => (*point, *point, true), + Atom::Between(low, high) => (*low, *high, false), + }; + + // Atoms are contiguous and in order, so one that runs the same arms as + // the one before it is the same table entry stretched further. + if run + .as_ref() + .map(|open| open.blocks == blocks) + .unwrap_or(false) + { + let open = run.as_mut().expect("just checked"); + open.to = to; + open.inclusive = inclusive; + continue; + } + + if let Some(open) = run.take() { + out.push(finish(open)); + } + // An atom no arm covers ends the run and starts nothing: a subject + // there has no range arm and belongs to the default. + if !blocks.is_empty() { + run = Some(Run { + from, + to, + inclusive, + blocks, + }); + } + } + + if let Some(open) = run.take() { + out.push(finish(open)); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `from..to => block`, as rhai's parser would record it. + fn exclusive(from: INT, to: INT, block: usize) -> RangeCase { + let mut case: RangeCase = (from..to).into(); + case.set_index(block); + case + } + + fn inclusive(from: INT, to: INT, block: usize) -> RangeCase { + let mut case: RangeCase = (from..=to).into(); + case.set_index(block); + case + } + + /// What the table says, as `(from, to, inclusive, blocks)`. + fn table(ranges: &[RangeCase]) -> Vec<(INT, INT, bool, Vec)> { + split(ranges) + .into_iter() + .map(|(range, blocks)| (range.from, range.to, range.inclusive, blocks)) + .collect() + } + + #[test] + fn nothing_in_gives_nothing_out() { + assert!(split(&[]).is_empty()); + } + + /// The common shape. Splitting must not turn two arms into six entries, + /// or every artifact with a `switch` in it pays for the case nobody + /// wrote. + #[test] + fn arms_that_do_not_overlap_come_back_unchanged() { + assert_eq!( + table(&[inclusive(0, 9, 0), inclusive(10, 99, 1)]), + vec![(0, 9, true, vec![0]), (10, 99, true, vec![1])], + ); + } + + /// Two arms over the same values — the shape that has no single answer at + /// runtime, and the reason this exists. + #[test] + fn identical_arms_become_one_entry_running_both() { + assert_eq!( + table(&[inclusive(0, 9, 0), inclusive(0, 9, 1)]), + vec![(0, 9, true, vec![0, 1])], + ); + } + + #[test] + fn a_partial_overlap_splits_into_three() { + assert_eq!( + table(&[exclusive(0, 10, 0), exclusive(5, 20, 1)]), + vec![ + (0, 5, false, vec![0]), + (5, 10, false, vec![0, 1]), + (10, 20, false, vec![1]), + ], + ); + } + + /// The awkward one: the arms meet at a single integer that only one of + /// them includes, and the piece above it cannot be named exactly. + #[test] + fn an_inclusive_end_meeting_an_exclusive_start_splits_at_the_point() { + let split = table(&[inclusive(0, 10, 0), exclusive(10, 20, 1)]); + + assert_eq!( + split, + vec![ + (0, 10, false, vec![0]), + (10, 10, true, vec![0, 1]), + (10, 20, false, vec![1]), + ], + ); + + // The last entry does contain `10`, which belongs to both arms — the + // entry before it is what makes that unreachable, so the order is + // part of the answer and not a presentation detail. + let (shared, ..) = &split[1]; + assert!(*shared == 10, "the point entry must come first"); + } + + /// A hole between two arms is not an entry: a subject in it has no range + /// arm at all and must reach the default. + #[test] + fn a_gap_between_arms_is_left_out() { + assert_eq!( + table(&[inclusive(0, 4, 0), inclusive(10, 14, 1)]), + vec![(0, 4, true, vec![0]), (10, 14, true, vec![1])], + ); + } + + /// An arm inside another arm keeps the outer one either side of it. + #[test] + fn a_nested_arm_splits_the_one_around_it() { + assert_eq!( + table(&[exclusive(0, 100, 0), exclusive(10, 20, 1)]), + vec![ + (0, 10, false, vec![0]), + (10, 20, false, vec![0, 1]), + (20, 100, false, vec![0]), + ], + ); + } + + /// Splitting only means anything if what comes out agrees with rhai about + /// which arms a value belongs to — so check the pieces against the arms + /// they came from, at every bound and between them. + #[test] + fn every_value_reaches_the_arms_rhai_would_have_run() { + let arms = [ + exclusive(0, 10, 0), + inclusive(5, 20, 1), + exclusive(20, 25, 2), + inclusive(-5, 0, 3), + ]; + let split = split(&arms); + + let mut probes: Vec = Vec::new(); + for value in -8..=28 { + probes.push(rhai::Dynamic::from(value as INT)); + // Halfway between two integers is where an interval that has no + // exact form goes wrong, so the integers alone would not find it. + #[cfg(not(feature = "no_float"))] + probes.push(rhai::Dynamic::from(value as rhai::FLOAT + 0.5)); + } + + for probe in &probes { + let expected: Vec = arms + .iter() + .filter(|arm| arm.contains(probe)) + .map(RangeCase::index) + .collect(); + + let found = split + .iter() + .find(|(range, ..)| range.contains(probe)) + .map(|(.., blocks)| blocks.clone()) + .unwrap_or_default(); + + assert_eq!(found, expected, "for {probe:?}"); + } + } +} diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs new file mode 100644 index 000000000..7a1707a04 --- /dev/null +++ b/src/grain/compile/mod.rs @@ -0,0 +1,2170 @@ +mod cases; +mod poolable; +mod slots; + +use core::mem; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +use crate::ast::{ + ASTFlags, Expr, FlowControl, FnCallExpr, OpAssignment, ScriptFuncDef, Stmt, StmtBlock, + SwitchCasesCollection, +}; +use crate::tokenizer::Token; +use crate::types::Span; +use crate::{Dynamic, ImmutableString, Position, AST}; + +use crate::grain::bytecode::{ + assemble, resolve_switch_targets, AssignOp, Chain, Chunk, Op, Positions, Receiver, Root, Step, + Switch, SwitchCase, SwitchRange, Tail, +}; +use crate::grain::compile::poolable::is_poolable; +use crate::grain::compile::slots::Slots; +use crate::grain::program::{Function, Parts, Program}; + +/// Whether a variable reference is module-qualified, as in `foo::bar`. +/// +/// `Expr::Variable`'s payload only carries a `Namespace` when modules are +/// compiled in. Under `no_module` the box is two fields rather than four and +/// nothing can be qualified, so the question has a constant answer and the +/// field it would have read does not exist. +#[cfg(not(feature = "no_module"))] +macro_rules! has_namespace { + ($payload:expr) => { + !$payload.2.is_empty() + }; +} +#[cfg(feature = "no_module")] +macro_rules! has_namespace { + ($payload:expr) => {{ + let _ = $payload; + false + }}; +} + +/// The same question for a call: is it `foo::bar()` rather than `bar()`. +/// `FnCallExpr` carries no `namespace` field at all under `no_module`. +#[cfg(not(feature = "no_module"))] +macro_rules! call_has_namespace { + ($call:expr) => { + !$call.namespace.is_empty() + }; +} +#[cfg(feature = "no_module")] +macro_rules! call_has_namespace { + ($call:expr) => {{ + let _ = $call; + false + }}; +} + +/// Lowers a rhai `AST` into a [`Program`]. +/// +/// Anything not yet lowered is kept as an AST fragment and handed back to +/// rhai's walker at runtime, so the output always means the same as its input. +/// Progress is [`Program::residual_count`] falling. +#[derive(Debug, Default, Clone)] +pub struct Compiler { + _private: (), +} + +impl Compiler { + /// Create a new [`Compiler`] with default options. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Lower an `AST` into a [`Program`]. + #[must_use] + pub fn compile(&self, ast: &AST) -> Program<'static> { + // A bare script-function name used as a value is not a variable read + // at all — rhai turns it into a function pointer with the calling + // environment attached (`eval/expr.rs:71-99`) — so those names must + // not become `LoadNamed`. Carried across every restart below, because + // function bodies are lowered after one. + let script_fns: Vec = ast + .shared_lib() + .iter_script_fn_info() + .map(|(.., def)| def.name.clone()) + .collect(); + let fresh = || Lowering { + script_fns: script_fns.clone(), + ..Lowering::default() + }; + + let mut lowering = fresh(); + + // Anything the slot model cannot account for costs the whole program + // its lowering rather than risking a scope it resolved slots against + // being a different shape at runtime. Coverage is preserved either way. + if !lowering.program(ast.statements(), true) { + lowering = fresh(); + lowering.whole_program_residual(ast.statements()); + } + let main_ops = lowering.code.len(); + + // Each function's body appends to the same instruction list, so the + // whole program assembles as one address space. A function the slot + // model cannot handle is simply left out, and rhai's own copy of it + // stays reachable through the library below. + let mut functions = Vec::new(); + let mut skipped = 0usize; + for (.., def) in ast.shared_lib().iter_script_fn_info() { + match lowering.function(def) { + Some(function) => functions.push(function), + None => skipped += 1, + } + } + + // Assembly can fail the same way the slot model can, for a script with + // more distinct names or constants than a `u16` operand can index — so + // it takes the same exit. The fallback is a single instruction and + // always assembles, which is what keeps coverage total. + // Switch targets are instruction indices too, and they live in the + // pool rather than in the code, so they are resolved separately — + // failing the same way, into the same fallback. + let assembled = match assemble(&lowering.code) { + Ok((code, offsets)) => resolve_switch_targets(&mut lowering.switches, &offsets) + .ok() + .map(|()| (code, offsets)), + Err(..) => None, + }; + let (code, offsets, main_ops, functions, skipped) = match assembled { + Some((code, offsets)) => (code, offsets, main_ops, functions, skipped), + None => { + lowering = fresh(); + lowering.whole_program_residual(ast.statements()); + let (code, offsets) = + assemble(&lowering.code).expect("the fallback is one instruction"); + (code, offsets, lowering.code.len(), Vec::new(), 1) + } + }; + + // Jump targets and the position table were both keyed on instruction + // index while lowering; instructions vary in length once assembled. + let mut positions = vec![rhai::Position::NONE; code.len()]; + for (index, pos) in lowering.positions.iter().enumerate() { + positions[offsets[index] as usize] = *pos; + } + + let main = Chunk::new(0, offsets[main_ops], lowering.max_stack); + let functions: Vec<_> = functions + .into_iter() + .map(|f| Function { + name: f.name, + params: f.params, + this_type: f.this_type, + // Derived from the chunk by `Program::new`, which is the one + // place that can see the assembled bytes. + takes_this: false, + chunk: Chunk::new( + offsets[f.first_op], + offsets[f.first_op + f.op_count], + lowering.max_stack, + ), + }) + .collect(); + + // rhai's own functions are carried whenever anything might still reach + // for them: a function this compiler skipped, or a fragment that could + // call one. With neither, every call resolves in the table above and + // the library — an `AST`'s whole function tree — can be dropped. + // + // The third case is a pointer to a `this`-taking chunk. Rhai reaches a + // compiled function through a registered wrapper, and a wrapper is + // registered at one arity — but a native calling a pointer against a + // receiver decides for itself how many arguments to append beside it, + // so no single arity is right. Rhai's own pointer carries the body and + // sizes the call from it, which is what its copy is kept here for. See + // `callback::wrappers`, which skips exactly these. + let escapes_as_pointer = crate::grain::program::makes_fn_pointers(&code) + && functions + .iter() + .any(|f| crate::grain::program::takes_this(&code, f.chunk, &lowering.chains)); + let needs_walker = skipped > 0 || !lowering.residuals.is_empty() || escapes_as_pointer; + + let mut program = Program::new( + code.into(), + main, + functions, + Parts { + positions: Positions::dense(positions), + residuals: lowering.residuals, + consts: lowering.consts, + names: crate::grain::bytecode::Strings::new(&lowering.names), + tokens: lowering.tokens, + assign_ops: lowering.assign_ops, + chains: lowering.chains, + switches: lowering.switches, + lib: (needs_walker && !ast.shared_lib().is_empty()) + .then(|| ast.shared_lib().clone()), + #[cfg(not(feature = "no_module"))] + resolver: ast.resolver.clone(), + source: ast.source().map(Into::into), + }, + ); + + // `max_stack` above is an upper bound the lowering can compute without + // a depth walk. The verifier does the walk anyway, so take its answer. + program.tighten_stack(); + program + } +} + +/// Where `break` and `continue` jump to, and what they must unwind first. +/// +/// Jump targets are backpatched: `break` sites are collected as they are +/// emitted and pointed at the instruction after the loop once that address is +/// known. +struct Loop { + /// Where `continue` goes — the condition test, or the top of the body. + continue_target: u32, + /// Slot depth a `break` unwinds to. For a `for` loop this is *before* the + /// loop variable, which leaving must drop. + break_depth: u16, + /// Slot depth a `continue` unwinds to. Differs from `break_depth` in a + /// `for`, where the loop variable has to survive into the next iteration — + /// one field cannot be both. + continue_depth: u16, + /// How many iterators are live *inside* this loop, so a jump out of it + /// can drop whatever was made since. A `break` inside a `try` inside a + /// `for` skips the straight-line path that would have cleaned up. + iters: usize, + /// Whether the loop owns an iterator of its own. `break` drops it and + /// `continue` must not, which is the other thing one field cannot be. + owns_iterator: bool, + /// How many `try` regions were armed when the loop began, so a jump out + /// of the loop disarms the ones inside it. + handlers: usize, + /// `Jump` sites awaiting the address after the loop. + breaks: Vec, +} + +/// Where a `switch` table entry sends control, before the arms have +/// addresses. +#[derive(Debug, Clone, Copy)] +enum Entry { + /// Straight to an arm's body: the group has no guard to try first. + Body(usize), + /// The head of a guard chain, which is already emitted. + At(u32), + /// Nothing in the group can run. + Default, +} + +/// A function body that lowered, before its instruction indices become byte +/// addresses. +struct LoweredFn { + name: u32, + params: Vec, + /// The declared receiver type, as a name-pool index. See + /// [`Function::this_type`](crate::grain::program::Function::this_type). + this_type: Option, + first_op: usize, + op_count: usize, +} + +#[derive(Default)] +struct Lowering { + code: Vec, + /// One per instruction, parallel to `code`. Most are `NONE`; the dense + /// shape is what makes a lookup an index, and it compacts on the way out. + positions: Vec, + residuals: Vec, + consts: Vec, + names: Vec, + tokens: Vec, + assign_ops: Vec, + chains: Vec, + switches: Vec, + slots: Slots, + max_stack: u16, + loops: Vec, + /// How many iterators are live at this point in the lowering, so a jump + /// out of a loop knows how many to drop. + iters: usize, + /// The same for `try` regions: a `break` out of one has to disarm it, or + /// the next unrelated error is caught into a block already left. + handlers: usize, + /// Names that are script functions rather than variables. + script_fns: Vec, + /// Set when something nested inside an expression defeated the slot model. + /// + /// [`Lowering::statement`] says so by returning false, but + /// [`Lowering::expression`] has no way to: it is called from the middle of + /// building other expressions, and every one of those callers would have + /// to thread the answer back. So a block used as an expression records the + /// failure here instead, and [`Lowering::program`] reports it. + /// + /// A sticky flag is enough because failure is all or nothing — the caller + /// throws the whole lowering away and starts again as one fragment — so + /// instructions emitted after it are discarded rather than run. + defeated: bool, +} + +impl Lowering { + /// Lower a statement list as a whole chunk. Returns false if something + /// defeated the slot model and the caller should fall back. + /// + /// `keeps_scope` says whether what this chunk declares outlives it, which + /// is true of the program and false of every function body. Only then is a + /// [`Op::Checkpoint`] worth emitting: it is what an escaping error unwinds + /// to, and a function's scope is discarded whole however it ends. + fn program(&mut self, statements: &[Stmt], keeps_scope: bool) -> bool { + let Some((last, leading)) = statements.split_last() else { + self.emit(Op::Unit); + self.emit(Op::Return); + return true; + }; + + for stmt in leading { + if keeps_scope { + self.emit(Op::Checkpoint); + } + if !self.statement(stmt) { + return false; + } + // A statement's value is only the program's value if it is the + // last one; rhai discards the rest. + self.emit(Op::Pop); + } + + if keeps_scope { + self.emit(Op::Checkpoint); + } + if !self.statement(last) { + return false; + } + + self.emit(Op::Return); + !self.defeated + } + + /// Lower `a.b[i].c`, either reading it or assigning to it. + /// + /// Returns false if the chain is not one this can express, in which case + /// the caller keeps it as a fragment. + /// + /// The shape is the awkward part. Rhai does not store a chain as a list: + /// `a.b[i]` is `Dot { lhs: a, rhs: Index { lhs: b, rhs: i } }`, where each + /// nested node's `lhs` is the *current* step's operand and its `rhs` is the + /// continuation. [`flatten_chain`] unpicks that into steps. + fn chain(&mut self, expr: &Expr, tail: Tail, value: Option<&Expr>) -> bool { + let Some((root, steps)) = flatten_chain(expr) else { + return false; + }; + + // A variable root is one the chain can write back into, by slot or by + // name; anything else has to be both a read and a value rhai would + // itself have evaluated into a temporary. + // + // `this` is deliberately not in the second class. Rhai reaches it + // through the caller's `&mut` (`eval/chaining.rs:528`), so a method + // step that mutates lands in the caller's value — walking a copy would + // drop the write silently. It gets a root of its own instead. + let root_spec = match root { + Expr::Variable(v, ..) if !has_namespace!(v) => match self.slots.resolve(&v.1) { + Some(slot) => Root::Local { + slot, + name: self.push_name(v.1.clone()), + }, + // The caller's, or a module's, or nothing — decided at run + // time, because which of the three it is decides whether the + // chain can write through it. + // + // The guard is load-bearing: a bare script-function name is a + // function pointer rather than a variable, and turning one + // into a name lookup would report it missing where rhai hands + // back a pointer. + None if self.is_variable_name(&v.1, false) => Root::Named { + name: self.push_name(v.1.clone()), + pos: root.position(), + }, + None => return false, + }, + Expr::ThisPtr(pos) => Root::This { pos: *pos }, + // A qualified root resolves against imported modules, which need + // `import` — the escape hatch's job. + Expr::Variable(..) => return false, + _ if matches!(tail, Tail::Read) => Root::Temporary, + // Unreachable through the parser, which refuses `f().x = 1` + // outright (`eval/chaining.rs:559`). + _ => return false, + }; + + // Index values and method arguments are evaluated first, in step + // order, exactly as rhai collects them before walking + // (`eval/chaining.rs:568`). Evaluating one partway down would need the + // operand stack while a borrow of the container is live. + let mut lowered = Vec::with_capacity(steps.len()); + let mut operands = 0u16; + + for step in &steps { + match step { + ChainStep::Index(index, bracket) => { + self.expression(index); + lowered.push(Step::Index { + operand: operands, + pos: index.start_position(), + bracket: *bracket, + }); + operands += 1; + } + ChainStep::Property(prop, pos) => { + let (getter, setter, name) = &**prop; + lowered.push(Step::Property { + name: self.push_name(name.clone()), + getter: self.push_name(getter.0.clone()), + setter: self.push_name(setter.0.clone()), + pos: *pos, + }); + } + ChainStep::Method(call, pos) => { + if !self.is_lowerable_call(call) { + return false; + } + let Ok(argc) = u8::try_from(call.args.len()) else { + return false; + }; + let first = operands; + for arg in call.args.iter() { + self.expression(arg); + operands += 1; + } + lowered.push(Step::Method { + name: self.push_name(call.name.clone()), + argc, + operand: first, + pos: *pos, + }); + } + } + } + + // Then the root, if it is one that has to be evaluated. After the + // operands rather than before, which is rhai's order and not the + // reading order: `[f()][g()]` calls `g` first. + if matches!(root_spec, Root::Temporary) { + self.expression(root); + } + + // The value being assigned goes on last, above everything, so the + // walk can take what it needs before it borrows the container. + if let Some(value) = value { + self.expression(value); + } + + let index = self.push_chain(Chain { + root: root_spec, + steps: lowered, + tail, + operands, + }); + self.emit_at(Op::Chain(index), expr.position()); + true + } + + /// Lower a `switch` into one dispatch table plus the arms it names. + /// + /// The layout is: the subject, [`Op::Switch`], the guard chains, the arm + /// bodies, the default. Every arm leaves one value and jumps to the end, + /// so the statement's value is the matched arm's — or unit, which is what + /// an absent `_` compiles to. + /// + /// Guards are why the table does not simply hold bodies. Rhai tries the + /// arms sharing a case value in source order and falls to the *default* + /// when they all decline — never on to the ranges (`eval/stmt.rs:544`) — + /// so a group becomes a chain of guards ending in a jump to the default, + /// and the table points at the chain. Nearly every arm anyone writes has + /// no guard, and those cost no chain at all. + fn switch(&mut self, subject: &Expr, sw: &SwitchCasesCollection) -> bool { + // Sorted because rhai's map iterates in whatever order its hasher put + // the entries in, and an artifact should not depend on that. + let mut groups: Vec<(u64, Vec)> = sw + .cases + .iter() + .map(|(hash, blocks)| (*hash, blocks.to_vec())) + .collect(); + groups.sort_unstable_by_key(|(hash, ..)| *hash); + + // Overlapping range arms have no single answer at runtime, so they are + // cut into disjoint pieces here instead. See [`cases::split`]. + let ranges = cases::split(&sw.ranges); + + // Unflattened: a shared subject is not hashable, and rhai's gate on + // that is what sends it to the default arm. Flattening here would make + // it match a case the walker skips. + self.unflattened(subject); + let table = self.push_switch(); + self.emit(Op::Switch(table)); + + // One chain per distinct list of arms, shared by every table entry + // naming it: `1 | 2 => ..` is two case values and one chain. + let mut chains: Vec<(&[usize], Entry)> = Vec::new(); + let mut to_body: Vec<(usize, usize)> = Vec::new(); + let mut to_default: Vec = Vec::new(); + + let lists = groups + .iter() + .map(|(.., blocks)| blocks.as_slice()) + .chain(ranges.iter().map(|(.., blocks)| blocks.as_slice())); + for blocks in lists { + if chains.iter().any(|(existing, ..)| *existing == blocks) { + continue; + } + let entry = self.arm_chain(sw, blocks, &mut to_body, &mut to_default); + chains.push((blocks, entry)); + } + + // Bodies, one per arm something can reach. An arm behind a constant + // false guard, or one whose range the parser dropped for being empty, + // is reachable by nothing and is not emitted. + let mut wanted: Vec = to_body.iter().map(|(.., block)| *block).collect(); + wanted.extend(chains.iter().filter_map(|(.., entry)| match entry { + Entry::Body(block) => Some(*block), + _ => None, + })); + wanted.extend(sw.def_case); + wanted.sort_unstable(); + wanted.dedup(); + + let mut body_at: Vec<(usize, u32)> = Vec::with_capacity(wanted.len()); + let mut to_end: Vec = Vec::with_capacity(wanted.len()); + for block in wanted { + body_at.push((block, self.here())); + // An arm body is an ordinary expression, and a block one goes + // through the same path as `let y = { .. }`. + self.expression(&sw.expressions[block].rhs); + if self.defeated { + return false; + } + to_end.push(self.emit_jump()); + } + + let at = |block: usize| { + body_at + .iter() + .find(|(candidate, ..)| *candidate == block) + .map(|(.., at)| *at) + .expect("every reachable arm was emitted above") + }; + + let default_at = match sw.def_case { + Some(block) => at(block), + None => { + let target = self.here(); + self.emit(Op::Unit); + target + } + }; + let end = self.here(); + + for site in to_end { + self.patch_to(site, end); + } + for site in to_default { + self.patch_to(site, default_at); + } + for (site, block) in to_body { + self.patch_to(site, at(block)); + } + + let target = |blocks: &[usize]| { + let entry = chains + .iter() + .find(|(existing, ..)| *existing == blocks) + .map(|(.., entry)| *entry) + .expect("every list got a chain above"); + match entry { + Entry::Body(block) => at(block), + Entry::At(target) => target, + Entry::Default => default_at, + } + }; + + self.switches[table as usize] = Switch { + cases: groups + .iter() + .map(|(hash, blocks)| SwitchCase { + hash: *hash, + target: target(blocks), + }) + .collect(), + ranges: ranges + .iter() + .map(|(range, blocks)| SwitchRange { + target: target(blocks), + ..*range + }) + .collect(), + default: default_at, + }; + + true + } + + /// Emit the guard chain for one group of arms, and say where the table + /// entries naming that group should point. + fn arm_chain( + &mut self, + sw: &SwitchCasesCollection, + blocks: &[usize], + to_body: &mut Vec<(usize, usize)>, + to_default: &mut Vec, + ) -> Entry { + let mut entry: Option = None; + + for block in blocks { + match &sw.expressions[*block].lhs { + // An arm without an `if` is a literal `true` in the tree + // (`parser.rs:1187`), so it always runs and everything after + // it in the group is unreachable. + Expr::BoolConstant(true, ..) => { + return match entry { + None => Entry::Body(*block), + Some(entry) => { + to_body.push((self.emit_jump(), *block)); + entry + } + }; + } + // Nothing can reach this arm, so nothing is emitted for it. + Expr::BoolConstant(false, ..) => continue, + guard => { + if entry.is_none() { + entry = Some(Entry::At(self.here())); + } + self.expression(guard); + let site = self.code.len(); + // Rhai reports a non-boolean guard against the guard, so + // the jump carries the guard's position. + self.emit_at(Op::JumpIfTrue { target: u32::MAX }, guard.position()); + to_body.push((site, *block)); + } + } + } + + match entry { + Some(entry) => { + to_default.push(self.emit_jump()); + entry + } + // Every arm in the group is behind a constant false guard, so the + // group is the default with extra steps. + None => Entry::Default, + } + } + + /// Reserve a table, to be filled in once its arms have addresses. + fn push_switch(&mut self) -> u32 { + self.switches.push(Switch { + cases: Vec::new(), + ranges: Vec::new(), + default: 0, + }); + (self.switches.len() - 1) as u32 + } + + fn push_chain(&mut self, chain: Chain) -> u32 { + if let Some(index) = self.chains.iter().position(|existing| *existing == chain) { + return index as u32; + } + self.chains.push(chain); + (self.chains.len() - 1) as u32 + } + + /// Lower one script function's body into the same instruction list. + /// + /// Returns `None` if the slot model cannot account for it, in which case + /// rhai keeps its own copy and calls to it go through dispatch. That is a + /// per-function decision: one awkward function does not cost the rest + /// their lowering. + /// + /// The body runs in a fresh scope with the parameters already pushed + /// (`func/script.rs:73`), so the parameters are exactly slots 0 upwards. + fn function(&mut self, def: &ScriptFuncDef) -> Option { + let first_op = self.code.len(); + let saved_slots = mem::take(&mut self.slots); + let saved_loops = mem::take(&mut self.loops); + // Per-function, like the slots: one body the model cannot handle must + // not cost the rest of the program its lowering. + let saved_defeated = mem::replace(&mut self.defeated, false); + + for param in def.params.iter() { + self.slots.declare(param.clone()); + } + let params: Vec<_> = def + .params + .iter() + .map(|p| self.push_name(p.clone())) + .collect(); + + // A body is a statement list whose last value is the return value, + // which is what `program` already does. + let lowered = self.program(def.body.statements(), false); + + self.slots = saved_slots; + self.loops = saved_loops; + self.defeated = saved_defeated; + + if !lowered { + // Roll back whatever the attempt emitted, so a function that could + // not be lowered leaves no unreachable instructions behind. + self.code.truncate(first_op); + self.positions.truncate(first_op); + return None; + } + + Some(LoweredFn { + name: self.push_name(def.name.clone()), + params, + this_type: def + .this_type + .as_ref() + .map(|typed| self.push_name(typed.clone())), + first_op, + op_count: self.code.len() - first_op, + }) + } + + /// The last-resort fallback: one fragment holding everything, evaluated + /// without rewinding so top-level declarations still reach the caller. + fn whole_program_residual(&mut self, statements: &[Stmt]) { + let body = wrap_statements(statements.to_vec()); + let residual = self.push_residual(body); + self.emit(Op::EvalAst { + residual, + rewind_scope: false, + }); + self.emit(Op::Return); + } + + /// Lower one statement, leaving its value on the stack. + fn statement(&mut self, stmt: &Stmt) -> bool { + match stmt { + Stmt::Var(payload, flags, ..) => { + // `export let x = ...` also binds a module alias, which the + // slot model does not represent. + if flags.intersects(ASTFlags::EXPORTED) || self.slots.is_full() { + return false; + } + + let (ident, init, ..) = &**payload; + self.expression(init); + + let name = self.push_name(ident.name.clone()); + self.slots.declare(ident.name.clone()); + self.emit(Op::DeclareLocal { + name, + is_const: flags.intersects(ASTFlags::CONSTANT), + }); + + // A declaration evaluates to unit. + self.emit(Op::Unit); + true + } + + Stmt::Expr(expr) => { + self.expression(expr); + true + } + + // Rhai gives a call standing alone as a statement its own node + // rather than wrapping it in `Stmt::Expr`, and an operator is a + // call — so without this every top-level `a * b` stayed a fragment. + // A closure's `curry` lands here rather than in `Stmt::Expr`, + // because rhai gives a call standing alone as a statement its own + // node. + Stmt::FnCall(call, pos) if self.fn_ptr_call(call, *pos) => true, + + Stmt::FnCall(call, pos) if self.is_lowerable_call(call) => { + self.lower_call(call, *pos); + true + } + + // Standing alone is the position `eval` is usually written in, and + // rhai gives it its own node — so this is the arm that catches it, + // not the `Expr::FnCall` one. See there for why it defeats the + // lowering rather than becoming a fragment. + Stmt::FnCall(call, ..) if call.name == crate::engine::KEYWORD_EVAL => false, + + // `this` on the left. Ahead of the two variable arms because rhai's + // parser puts it there too (`parser.rs:2002`), and because the + // chain arm below would otherwise take `this.x = 1`'s sibling. + Stmt::Assignment(payload) if matches!(&payload.1.lhs, Expr::ThisPtr(..)) => { + let (op_info, binary) = &**payload; + + // Before the right-hand side, not after. Rhai checks that + // `this` is bound and returns before it evaluates the value + // (`eval/stmt.rs:300-303`) — unlike the variable arm, which + // evaluates first — so an unbound `this = nosuch` is + // `ErrorUnboundThis` and not the value's own failure. + self.emit_at(Op::RequireThis, binary.lhs.position()); + + self.expression(&binary.rhs); + let op = self.op_assignment(op_info); + + self.emit_at(Op::AssignThis { op }, op_info.position()); + self.emit(Op::Unit); + true + } + + // A plain local on the left. + Stmt::Assignment(payload) + if matches!(&payload.1.lhs, Expr::Variable(v, ..) + if !has_namespace!(v) && self.slots.resolve(&v.1).is_some()) => + { + let (op_info, binary) = &**payload; + let Expr::Variable(v, ..) = &binary.lhs else { + unreachable!("checked by the guard"); + }; + let slot = self.slots.resolve(&v.1).expect("checked by the guard"); + let var_name = self.push_name(v.1.clone()); + + self.expression(&binary.rhs); + let op = self.op_assignment(op_info); + + self.emit_at(Op::AssignLocal { slot, var_name, op }, op_info.position()); + self.emit(Op::Unit); + true + } + + // A variable no slot names — the caller's. Same shape as above, + // and the same op-assignment resolution; only where the target + // lives differs. + Stmt::Assignment(payload) + if matches!(&payload.1.lhs, Expr::Variable(v, ..) + if self.is_variable_name(&v.1, has_namespace!(v))) => + { + let (op_info, binary) = &**payload; + let Expr::Variable(v, ..) = &binary.lhs else { + unreachable!("checked by the guard"); + }; + let name = self.push_name(v.1.clone()); + + self.expression(&binary.rhs); + let op = self.op_assignment(op_info); + + // The variable's position, not the operator's — unlike + // `AssignLocal`. The errors this instruction raises itself are + // `ErrorAssignmentToConstant` and `ErrorVariableNotFound`, and + // rhai reports both against the variable (`eval/stmt.rs:340` + // and `eval/stmt.rs:120`). For a local those are unreachable, + // because the parser rejects a constant it can see; for a name + // the caller supplied they are the common failures. + self.emit_at(Op::AssignNamed { name, op }, binary.lhs.position()); + self.emit(Op::Unit); + true + } + + // A chain on the left. The value goes on the stack after the + // chain's own operands, so the walk has everything it needs before + // it takes a borrow of the container. + Stmt::Assignment(payload) + if matches!(&payload.1.lhs, Expr::Dot(..) | Expr::Index(..)) => + { + let (op_info, binary) = &**payload; + let op = self.op_assignment(op_info); + + let mark = self.mark(); + if !self.chain(&binary.lhs, Tail::Assign { op }, Some(&binary.rhs)) { + self.rewind(mark); + let residual = self.push_residual(wrap_statements(vec![stmt.clone()])); + self.emit(Op::EvalAst { + residual, + rewind_scope: true, + }); + } + // The chain leaves unit, which is what an assignment evaluates + // to, so there is nothing to add here. + true + } + + // Emitted by the parser ahead of the `curry` call that binds a + // closure's captures (`parser.rs:3707`). + #[cfg(not(feature = "no_closure"))] + Stmt::Share(names) => { + for (ident, ..) in names.iter() { + match self.slots.resolve(&ident.name) { + Some(slot) => self.emit_at(Op::Share(slot), ident.pos), + None => { + // The caller's — a closure can capture something + // no slot addresses. + let name = self.push_name(ident.name.clone()); + self.emit_at(Op::ShareNamed(name), ident.pos); + } + } + } + self.emit(Op::Unit); + true + } + + Stmt::Block(block) => self.block(block.statements()), + + // `try { .. } catch (e) { .. }`. + // + // The catch block's value is thrown away: rhai's whole statement + // is the try block's value on the way through and *unit* when + // something was caught (`.map(|_| Dynamic::UNIT)`, + // `eval/stmt.rs:863`). So `try { throw 7 } catch (e) { e * 2 }` is + // unit, not 14. + Stmt::TryCatch(payload, ..) => { + let FlowControl { expr, body, branch } = &**payload; + + // An absent catch variable is `Expr::Unit`; a present one is + // an `Expr::Variable` whose position is what rhai reports + // `ErrorTooManyVariables` against. + let catch_var = match expr { + Expr::Variable(v, ..) => Some(v.1.clone()), + _ => None, + }; + + let catch_name = catch_var.clone().map(|name| self.push_name(name)); + let site = self.code.len(); + self.emit_at( + Op::PushHandler { + target: u32::MAX, + catch_var: catch_name, + }, + expr.position(), + ); + self.handlers += 1; + + if !self.block(body.statements()) { + return false; + } + self.emit(Op::PopHandler); + self.handlers -= 1; + let past = self.emit_jump(); + + // The catch block, entered with the scope back where the `try` + // began and the variable already pushed on top of it. The + // handler is still armed here — that is what makes a bare + // `throw;` in this block a re-raise — so the depth goes back + // up, and the `PopHandler` below is what ends the region. + self.patch_to(site, self.here()); + self.handlers += 1; + let depth = self.slots.depth(); + if let Some(name) = catch_var { + self.slots.declare(name); + } + if !self.block(branch.statements()) { + return false; + } + self.emit(Op::Pop); + self.unwind_to(depth); + self.emit(Op::PopHandler); + self.handlers -= 1; + self.emit(Op::Unit); + + self.patch_here(past); + true + } + + // `for x in seq` / `for (x, i) in seq`. + // + // The loop variable and counter are pushed once and written each + // time round, not re-pushed — rhai does the same (`stmt.rs:708`), + // and it is observable: a closure made in the body captures the + // cell, so every one of them sees the last value. + Stmt::For(payload, ..) => { + let (var, counter, flow) = &**payload; + let outside = u16::try_from(self.slots.depth()).expect("slot count is bounded"); + + self.expression(&flow.expr); + // `ErrorFor` is reported against the iterable's *start*, which + // for `a.b` or a call is not its `position`. + self.emit_at(Op::IterInit, flow.expr.start_position()); + self.iters += 1; + + // Counter first, matching the order rhai pushes them in, so + // the slots line up with the scope it builds. + let counter_slot = counter.as_ref().map(|ident| { + let name = self.push_name(ident.name.clone()); + self.emit(Op::Unit); + self.emit(Op::DeclareLocal { + name, + is_const: false, + }); + self.slots.declare(ident.name.clone()); + self.slots.depth() as u16 - 1 + }); + let var_name = self.push_name(var.name.clone()); + self.emit(Op::Unit); + self.emit(Op::DeclareLocal { + name: var_name, + is_const: false, + }); + self.slots.declare(var.name.clone()); + let var_slot = self.slots.depth() as u16 - 1; + + let top = self.here(); + let exit = self.code.len(); + self.emit_at( + Op::IterNext { + exit: u32::MAX, + indexed: counter_slot.is_some(), + }, + flow.expr.position(), + ); + // The item is on top, the count under it, so these pop in the + // order the two locals were declared. + self.emit(Op::StoreShared(var_slot)); + if let Some(slot) = counter_slot { + self.emit(Op::StoreShared(slot)); + } + self.emit_at(Op::Tick, flow.body.position()); + + self.begin_for(top, outside); + if !self.block_discarding(flow.body.statements()) { + return false; + } + self.emit(Op::Jump(top)); + let breaks = self.end_loop(); + + // Exhausted: `IterNext` dropped the iterator on the way here. + self.patch_to(exit, self.here()); + self.iters -= 1; + self.emit(Op::UnwindTo(outside)); + self.slots.unwind_to(outside as usize); + + self.emit(Op::Unit); + let past = self.emit_jump(); + for site in breaks { + self.patch_here(site); + } + self.patch_here(past); + true + } + + Stmt::Switch(payload, ..) => { + let (subject, cases) = &**payload; + self.switch(subject, cases) + } + + Stmt::If(payload, ..) => { + let FlowControl { expr, body, branch } = &**payload; + + self.expression(expr); + let to_else = self.emit_jump_if_false(expr.position()); + + if !self.block(body.statements()) { + return false; + } + let past_else = self.emit_jump(); + + self.patch_here(to_else); + if !self.block(branch.statements()) { + return false; + } + self.patch_here(past_else); + true + } + + // `loop` and `while true` are the same node: rhai marks an + // unconditional loop with a unit or `true` guard + // (`eval/stmt.rs:575-576`). + Stmt::While(payload, ..) => { + let FlowControl { expr, body, .. } = &**payload; + let unconditional = matches!(expr, Expr::Unit(..) | Expr::BoolConstant(true, ..)); + + let top = self.here(); + self.emit_at(Op::Tick, body.position()); + + let exit = if unconditional { + None + } else { + self.expression(expr); + Some(self.emit_jump_if_false(expr.position())) + }; + + self.begin_loop(top); + if !self.block_discarding(body.statements()) { + return false; + } + self.emit(Op::Jump(top)); + + let breaks = self.end_loop(); + if let Some(exit) = exit { + self.patch_here(exit); + } + // A `while` that runs to completion is unit; a `break value` + // supplies its own. Both arrive here with the stack balanced. + self.emit(Op::Unit); + let past = self.emit_jump(); + for site in breaks { + self.patch_here(site); + } + self.patch_here(past); + true + } + + Stmt::Do(payload, flags, ..) => { + let FlowControl { expr, body, .. } = &**payload; + let until = flags.intersects(ASTFlags::NEGATED); + + let top = self.here(); + self.emit_at(Op::Tick, body.position()); + + self.begin_loop(top); + if !self.block_discarding(body.statements()) { + return false; + } + let breaks = self.end_loop(); + + self.expression(expr); + if until { + // `do ... until c` loops while `c` is false, which is a + // false-jump straight back to the top. + self.emit_at(Op::JumpIfFalse { target: top }, expr.position()); + } else { + let exit = self.emit_jump_if_false(expr.position()); + self.emit(Op::Jump(top)); + self.patch_here(exit); + } + + self.emit(Op::Unit); + let past = self.emit_jump(); + for site in breaks { + self.patch_here(site); + } + self.patch_here(past); + true + } + + Stmt::BreakLoop(value, flags, ..) => { + let Some(active) = self.loops.last() else { + // Outside any loop this is a parse error in rhai, so it + // should be unreachable; bail rather than emit a jump to + // nowhere. + return false; + }; + let continue_target = active.continue_target; + let loop_iters = active.iters; + let loop_handlers = active.handlers; + let owns_iterator = active.owns_iterator; + let (break_depth, continue_depth) = (active.break_depth, active.continue_depth); + let is_break = flags.intersects(ASTFlags::BREAK); + + // A jump out of a loop skips whatever the straight-line path + // would have cleaned up. The nesting is lexical, so how many + // iterators are live is known here — a `break` inside a `try` + // inside a `for` has one to drop, and `continue` has none + // because it re-enters the loop that owns it. + if is_break { + match value { + Some(expr) => self.expression(expr), + None => self.emit(Op::Unit), + } + // Out of the loop entirely, so its own iterator goes too — + // `loop_iters` counts from inside the loop and therefore + // already includes it. + self.pop_handlers(loop_handlers); + self.drop_iterators(loop_iters - usize::from(owns_iterator)); + self.emit(Op::UnwindTo(break_depth)); + let site = self.emit_jump(); + self.loops.last_mut().expect("checked").breaks.push(site); + } else { + // Back into the same loop, so its iterator and its loop + // variable both have to survive. + self.pop_handlers(loop_handlers); + self.drop_iterators(loop_iters); + self.emit(Op::UnwindTo(continue_depth)); + self.emit(Op::Jump(continue_target)); + } + + // Unreachable, but every statement must leave a value for the + // caller's `Pop`, and the verifier checks depth on every path. + self.emit(Op::Unit); + true + } + + // `throw` shares this node, flagged, and unwinds as an error + // rather than returning. The position is the keyword's, not the + // expression's (`eval/stmt.rs:877`). + Stmt::Return(value, flags, pos) if flags.intersects(ASTFlags::BREAK) => { + match value { + Some(expr) => self.expression(expr), + None => self.emit(Op::Unit), + } + self.emit_at(Op::Throw, *pos); + // Unreachable, but every statement leaves a value for the + // caller's `Pop` and the verifier checks depth on every path. + self.emit(Op::Unit); + true + } + + Stmt::Return(value, flags, ..) if !flags.intersects(ASTFlags::BREAK) => { + match value { + Some(expr) => self.expression(expr), + None => self.emit(Op::Unit), + } + self.emit(Op::Return); + self.emit(Op::Unit); + true + } + + // The one statement the fragment fallback below cannot hold. + // + // `import` declares into the imports stack rather than the scope, + // and a fragment that rewinds truncates that stack on the way out + // (`eval/stmt.rs:55`) — so the alias would be gone before the next + // statement could name it, and a qualified call is its own + // fragment. Refusing the lowering hands the body to the walker + // whole, which is where the alias lives long enough to be used. + #[cfg(not(feature = "no_module"))] + Stmt::Import(..) => false, + + // Not lowered yet, and listed rather than matched with `_` on + // purpose. A wildcard here silently turned `import` and `eval` + // into fragments that answered differently from the walker; naming + // every kind means a new one added to rhai's AST stops the build + // until someone has decided which of the three it is — lowered, + // fragment, or too scope-shaped to be either. + // + // The ones below are fragments because each either declares + // nothing or rewinds what it declares, so the scope is the same + // shape afterwards. That is the property to check before adding to + // this list. + other @ (Stmt::Noop(..) + | Stmt::FnCall(..) + | Stmt::Assignment(..) + | Stmt::Return(..)) => { + let residual = self.push_residual(wrap_statements(vec![other.clone()])); + self.emit(Op::EvalAst { + residual, + rewind_scope: true, + }); + true + } + + #[cfg(not(feature = "no_module"))] + other @ Stmt::Export(..) => { + let residual = self.push_residual(wrap_statements(vec![other.clone()])); + self.emit(Op::EvalAst { + residual, + rewind_scope: true, + }); + true + } + } + } + + /// Lower one expression, leaving its value on the stack. + fn expression(&mut self, expr: &Expr) { + match expr { + Expr::BoolConstant(value, ..) => self.emit(Op::Bool(*value)), + Expr::Unit(..) => self.emit(Op::Unit), + + Expr::IntegerConstant(value, ..) => self.constant(Dynamic::from(*value)), + Expr::CharConstant(value, ..) => self.constant(Dynamic::from(*value)), + Expr::StringConstant(value, ..) => self.constant(Dynamic::from(value.clone())), + // Rhai has no float literal to parse under `no_float`, so there is + // no variant to match. + #[cfg(not(feature = "no_float"))] + Expr::FloatConstant(value, ..) => self.constant(Dynamic::from(**value)), + // Folded by the optimizer, so it can hold anything a constant call + // returned — including a function pointer, which must not be + // copied out of a pool. See `poolable`. + // A function pointer the optimizer folded — `Fn("f")` with a + // constant name, or a closure literal. It cannot go in the pool: + // a closure's carries a `ScriptFuncDef`, which is an AST body and + // exactly what an artifact must not contain. Rebuilt by name + // instead, which reaches the chunk we compiled from that same + // body. + // + // A constant function pointer: a closure literal, or what the + // optimizer folds `Fn("f")` into. Either way it embeds a + // `ScriptFuncDef` — an AST body, `Fn*` in rhai's own rendering — + // so it cannot go in the pool. Rebuilt by name instead, reaching + // the chunk compiled from that same body. + // + // There is no version of this that keeps the rendering: the thing + // that differs *is* the tree, and carrying it is what an artifact + // must not do. `a_closure_pointer_is_late_bound` pins the + // difference for both spellings. + // + // Curried values are arbitrary `Dynamic`s with the same problem one + // level down, and are left to the walker. + Expr::DynamicConstant(value, ..) + if value + .read_lock::() + .map(|f| f.curry().is_empty()) + .unwrap_or(false) => + { + let name = value + .read_lock::() + .expect("checked by the guard") + .fn_name() + .to_string(); + let name = self.push_name(name.into()); + self.emit_at(Op::MakeClosure(name), expr.position()); + } + + Expr::DynamicConstant(value, ..) if is_poolable(value) => { + self.constant((**value).clone()); + } + + Expr::Variable(payload, ..) => { + // A qualified name resolves against imported modules, not the + // scope, so it is not a slot. + let is_qualified = has_namespace!(payload); + + match self.slots.resolve(&payload.1) { + Some(slot) if !is_qualified => self.emit(Op::LoadLocal(slot)), + // Not a local this compiler declared, so no slot can name + // it: it is the caller's, a module's, or nothing. Looked + // up by name at run time, at the cost of a scope scan. + _ if self.is_variable_name(&payload.1, is_qualified) => { + let name = self.push_name(payload.1.clone()); + self.emit_at(Op::LoadNamed(name), expr.position()); + } + // A qualified name resolves against imported modules, and + // a bare function name is a function pointer. Neither is a + // variable read, and both stay rhai's job. + _ => self.residual_expr(expr), + } + } + + Expr::And(operands, ..) => self.short_circuit(operands, false), + Expr::Or(operands, ..) => self.short_circuit(operands, true), + + Expr::FnCall(call, pos) if self.fn_ptr_call(call, *pos) => {} + + Expr::FnCall(call, pos) if self.is_lowerable_call(call) => { + self.lower_call(call, *pos); + } + + // `eval` evaluates a script in the *caller's* scope, so what it + // declares outlives it and the next statement can name it. The + // slot model resolved its indices against a scope that does not + // have those entries, so a lowered read past an `eval` looks in + // the wrong place — `eval("let x = 40"); x + 2` found no `x` where + // the walker found 40. Refusing the lowering hands the body to the + // walker, which is the only thing that knows the real shape. + Expr::FnCall(call, ..) if call.name == crate::engine::KEYWORD_EVAL => { + self.residual_expr(expr); + self.defeated = true; + } + + // A literal whose elements are all constant never reaches here — + // rhai's optimizer folds it into a `DynamicConstant` first — so + // this is the one that has to be built at run time. + Expr::Array(elements, ..) if elements.len() <= u16::MAX as usize => { + for (index, element) in elements.iter().enumerate() { + self.expression(element); + // Positioned at the element, because that is what rhai + // blames when this element is the one that tips the + // running total over the limit. + self.emit_at( + Op::CheckSize { + index: index as u16, + map: false, + }, + element.position(), + ); + } + self.emit_at(Op::MakeArray(elements.len() as u16), expr.position()); + } + + // The other half of the same shape. Rhai keeps a map literal as a + // template holding every key — the constant values already in + // place, the computed ones as placeholders — plus the list of + // entries still to evaluate (`ast/expr.rs:283`). An all-constant + // map is folded into a `DynamicConstant` and never arrives here; + // one with a single computed value does, and used to fragment. + Expr::Map(entries, ..) if entries.0.len() <= u16::MAX as usize => { + let (computed, template) = &**entries; + let template = Dynamic::from_map(template.clone()); + // A template whose constants the pool cannot hold is a program + // that could not be written to an artifact anyway. + if !is_poolable(&template) { + self.residual_expr(expr); + return; + } + + self.constant(template); + for (index, (key, value)) in computed.iter().enumerate() { + self.constant(key.name.clone().into()); + self.expression(value); + self.emit_at( + Op::CheckSize { + index: index as u16, + map: true, + }, + value.position(), + ); + } + self.emit_at(Op::MakeMap(computed.len() as u16), expr.position()); + } + + // A block used for its value: `let y = if c { 1 } else { 2 }`, + // `let y = switch ..`, `let y = { let z = 1; z }`. Rhai evaluates + // it with `restore_orig_state` set (`eval/expr.rs:434`), so it + // rewinds what it declared — which is what `block` emits. + Expr::Stmt(block) => { + if !self.block(block.statements()) { + self.defeated = true; + } + } + + // The optimizer folds an all-constant interpolation away before + // this sees it, so what arrives has at least two segments. + Expr::InterpolatedString(segments, ..) => { + self.emit(Op::InterpolateStart); + for segment in segments.iter() { + self.expression(segment); + // The append carries the segment's own position, because + // that is what rhai blames when the size limit goes over. + self.emit_at(Op::InterpolateAppend, segment.position()); + } + self.emit(Op::InterpolateEnd); + } + + // `f.call(x)` and `f.curry(x)` are the method spellings of the two + // above. They arrive as chains, so they have to be taken before + // the chain walker sees them. + // An `rhs` that is a bare `MethodCall` is the whole chain: a + // further step would make it a `Dot` or an `Index` instead. + Expr::Dot(binary, ..) + if matches!(&binary.rhs, Expr::MethodCall(m, ..) + if matches!(m.name.as_str(), "call" | "curry") + && m.args.len() <= u8::MAX as usize) => + { + let Expr::MethodCall(method, ..) = &binary.rhs else { + unreachable!("checked by the guard"); + }; + // `obj.call(f)` binds `obj` as the closure's `this` by + // reference (`func/call.rs:862`), so a write inside the closure + // has to reach `obj`. The value goes on the stack as it always + // did; the receiver says where to carry a write back to. + // + // Unflattened, for the reason `unflattened` gives: a receiver + // that is a shared cell has to arrive *as* the cell, so a write + // lands where every holder can see it and no write-back is + // needed at all. + let receiver = self.fn_ptr_receiver(&binary.lhs); + if receiver.is_some() { + self.unflattened(&binary.lhs); + } else { + self.expression(&binary.lhs); + } + for arg in method.args.iter() { + self.expression(arg); + } + let argc = method.args.len() as u8; + // The call's own position, which is what rhai reports for + // everything the pointer path can raise. The one case it is + // not is `obj.call(x)` where `obj` is not a pointer and `x` is + // taken as one: rhai blames `x` (`func/call.rs:838`). Both + // cannot come from one position-table entry, and using the + // argument's instead was measured to move the divergence onto + // the common path rather than remove it. + // + // Method style only. `curry(f, ..)` written as a call is a + // different path in rhai and takes the *argument's* position — + // see `fn_ptr_call`. The two disagreeing is deliberate. + let pos = binary.rhs.position(); + if method.name == "call" { + self.emit_at( + Op::CallFnPtr { + argc, + method: true, + receiver, + }, + pos, + ); + } else { + self.emit_at(Op::Curry(argc), pos); + } + } + + Expr::Dot(..) | Expr::Index(..) => { + // A chain emits its own operands, so a failed attempt has to + // leave nothing behind. + let mark = self.mark(); + if !self.chain(expr, Tail::Read, None) { + self.rewind(mark); + self.residual_expr(expr); + } + } + + // Custom syntax runs host code against an `EvalContext`, which can + // declare into the caller's scope. What it declares is invisible + // here, so the slot model would be resolved against a scope shape + // that is not the one at runtime. Refusing the lowering keeps the + // walker's answer, as it does for `eval` above. + #[cfg(not(feature = "no_custom_syntax"))] + Expr::Custom(..) => { + self.residual_expr(expr); + self.defeated = true; + } + + // Listed rather than matched with `_`, for the reason + // [`Lowering::statement`] gives: a wildcard is what let `eval` + // become a fragment that answered differently from the walker. + // + // These are fragments because none of them can change the shape of + // the scope the slot model resolved its indices against. The + // guarded arms above fall through to here when their guard fails — + // a pool-defeating constant, a literal too long for its operand, a + // call rhai resolves syntactically. + // The frame's receiver, flattened as every consumer but three + // wants it — see [`Op::LoadThis`] and `unflattened` below. Its own + // position, because that is what `ErrorUnboundThis` carries. + Expr::ThisPtr(pos) => self.emit_at(Op::LoadThis, *pos), + + Expr::Coalesce(..) + | Expr::MethodCall(..) + | Expr::Property(..) + | Expr::DynamicConstant(..) + | Expr::FnCall(..) + | Expr::Array(..) + | Expr::Map(..) => self.residual_expr(expr), + } + } + + /// Where `obj.call(f)`'s receiver came from, when a write through the + /// closure's `this` has somewhere to land. + /// + /// `None` for anything rhai would evaluate into a temporary — `[1, 2].call(f)` + /// mutates a copy in the walker too, so there is nothing to carry back. + fn fn_ptr_receiver(&mut self, receiver: &Expr) -> Option { + match receiver { + Expr::ThisPtr(..) => Some(Receiver::This), + Expr::Variable(payload, ..) if !has_namespace!(payload) => { + match self.slots.resolve(&payload.1) { + Some(slot) => Some(Receiver::Local(slot)), + None if self.is_variable_name(&payload.1, false) => { + Some(Receiver::Named(self.push_name(payload.1.clone()))) + } + None => None, + } + } + _ => None, + } + } + + /// Where rhai's method-call rewrite would take this call's first argument + /// from, if it applies at all (`func/call.rs:1434`). + fn receiver(&mut self, call: &FnCallExpr) -> Option { + // An operator short-circuits before the rewrite is reached, and a call + // that captures the enclosing scope is excluded from it outright + // (`func/call.rs:1387` and `:1775`). + if call.op_token.is_some() || call.capture_parent_scope { + return None; + } + + // `f(this, ..)` takes the same rewrite as a variable. Rhai also requires + // the receiver not to be shared and nothing to be curried + // (`func/call.rs:1417`), and neither is a question the compiler can + // answer: sharing is a run-time property, deferred to the VM as it + // already is for a read-only local, and a curried redirect can never + // reach this instruction because `call`/`curry` go through + // `Op::CallFnPtr` and `is_lowerable_call` refuses them here. + if let Some(Expr::ThisPtr(..)) = call.args.first() { + return Some(Receiver::This); + } + + let Some(Expr::Variable(payload, ..)) = call.args.first() else { + return None; + }; + let qualified = has_namespace!(payload); + + match self.slots.resolve(&payload.1) { + Some(slot) if !qualified => Some(Receiver::Local(slot)), + _ if self.is_variable_name(&payload.1, qualified) => { + Some(Receiver::Named(self.push_name(payload.1.clone()))) + } + _ => None, + } + } + + /// Push the arguments left to right, then dispatch. + fn lower_call(&mut self, call: &FnCallExpr, pos: Position) { + let argc = u8::try_from(call.args.len()).expect("checked by is_lowerable_call"); + + // `f(x, ..)` is `x.f(..)`, so the variable is read after the other + // arguments and by reference. See [`Op::CallRef`]. + if let Some(receiver) = self.receiver(call) { + // `this` goes on *first*, unlike either of the others. Rhai's two + // arms disagree about when it is read: the by-reference one takes a + // pointer after the arguments (`func/call.rs:1417`), but the + // fallback a shared or unbound receiver lands in reads and flattens + // it before them (`:1462`). Reading first is what makes an unbound + // `f(this, nosuch)` report `ErrorUnboundThis`, and what stops an + // argument that writes to `this` being seen by the value passed. + if let Receiver::This = receiver { + self.emit_at(Op::LoadThis, call.args[0].position()); + } + for arg in call.args.iter().skip(1) { + self.expression(arg); + } + // A name is resolved here, where its own position is the one an + // `ErrorVariableNotFound` wants, and then moved under the arguments + // it was read after. + if let Receiver::Named(var) = receiver { + self.emit_at(Op::LoadNamed(var), call.args[0].position()); + if argc > 1 { + self.emit(Op::Rotate(argc - 1)); + } + } + + let name = self.push_name(call.name.clone()); + self.emit_at( + Op::CallRef { + name, + argc, + receiver, + }, + pos, + ); + return; + } + + for arg in call.args.iter() { + self.expression(arg); + } + let name = self.push_name(call.name.clone()); + // Only for a binary operator, which is the only shape the built-in + // lookup takes. Keeping a unary one would be dead weight and worse: + // `UnaryMinus` and `Minus` share the syntax `"-"`, so it is a token + // that cannot be written to an artifact at all. + let op = (argc == 2) + .then(|| call.op_token.clone()) + .flatten() + .map(|token| self.push_token(token)); + self.emit_at(Op::Call { name, argc, op }, pos); + } + + /// Whether a name read is a variable read at all. + /// + /// A qualified name resolves against imported modules rather than the + /// scope, and a bare script-function name is a function pointer with the + /// calling environment attached (`eval/expr.rs:71-99`). Neither is + /// something to look up by name, and both stay fragments. + fn is_variable_name(&self, name: &ImmutableString, qualified: bool) -> bool { + !qualified && !self.script_fns.contains(name) + } + + /// Pool what `x op= y` needs, if there is an operator at all. + fn op_assignment(&mut self, op_info: &OpAssignment) -> Option { + op_info + .get_op_assignment_info() + .map(|(_, _, op_assign, op_assign_str, op, op_str)| { + let entry = AssignOp { + op_assign: op_assign.clone(), + op_assign_name: self.push_name(op_assign_str.into()), + op: op.clone(), + op_name: self.push_name(op_str.into()), + }; + self.push_assign_op(entry) + }) + } + + /// Read a variable without flattening it, leaving a shared cell shared. + /// + /// Rhai's own variable read works this way — `Target::take_or_clone` hands + /// back the shared value untouched (`eval/target.rs:233`) — and the places + /// that want the contents flatten for themselves. [`Op::LoadLocal`] + /// flattens instead, which is right where the value is what matters and + /// wrong in the two places the cell is: + /// + /// * a closure's captured variable, where the aliasing *is* the capture; + /// * a `switch` subject, which rhai refuses to match on when it is not + /// hashable, and a shared value is not — so a shared subject falls to the + /// default arm however well it would otherwise have matched. + fn unflattened(&mut self, expr: &Expr) { + match expr { + Expr::Variable(payload, ..) if !has_namespace!(payload) => { + match self.slots.resolve(&payload.1) { + Some(slot) => self.emit(Op::LoadShared(slot)), + // The caller's. A closure can capture one of those too, and + // reading it flat would bind a copy. + None if self.is_variable_name(&payload.1, false) => { + let name = self.push_name(payload.1.clone()); + self.emit_at(Op::LoadSharedNamed(name), expr.position()); + } + None => self.expression(expr), + } + } + // The receiver can be a shared cell too — a closure capturing the + // variable a method was called on — and the three readers that come + // through here have to see the cell rather than what it holds. + Expr::ThisPtr(pos) => self.emit_at(Op::LoadThisShared, *pos), + other => self.expression(other), + } + } + + /// Lower `Fn(name)`, `curry(f, ..)` or `call(f, ..)`, if this is one. + /// + /// Rhai resolves these three by name before dispatch, but only at the + /// arities it recognises (`func/call.rs:1109-1245`); anything else is an + /// ordinary call that will not find a function. Matching those arities + /// exactly is what keeps the two agreeing on the failures as well as the + /// successes. + fn fn_ptr_call(&mut self, call: &FnCallExpr, pos: Position) -> bool { + if call_has_namespace!(call) || call.capture_parent_scope { + return false; + } + let argc = call.args.len(); + + match (call.name.as_str(), argc) { + // The argument has to arrive as the cell, not its contents, or the + // answer is always false. + // + // Not lowered under `no_closure`: rhai registers no `is_shared` + // there, so the call has to reach the walker and fail the way rhai + // fails it. Lowering it would answer a question rhai refuses. + #[cfg(not(feature = "no_closure"))] + ("is_shared", 1) => { + self.unflattened(&call.args[0]); + self.emit_at(Op::IsShared, pos); + } + // Both of these are reported against the *argument* rather than + // against the call: rhai reads it, and everything it can then + // complain about — a name that is not a string, a string that is + // not an identifier, a first argument that is not a pointer — is + // filled in with the argument's position (`func/call.rs:1217`, + // `:1220`, `:1232`). + ("Fn", 1) => { + self.expression(&call.args[0]); + self.emit_at(Op::MakeFnPtr, call.args[0].position()); + } + ("curry", _) if argc > 1 => { + let mut args = call.args.iter(); + self.expression(args.next().expect("checked by the arity")); + for arg in args { + // The captured variables. These must bind the *cell* — a + // flattening read would hand the closure a copy and it + // would stop being one. + self.unflattened(arg); + } + self.emit_at(Op::Curry((argc - 1) as u8), call.args[0].position()); + } + ("call", _) if argc >= 1 && argc <= u8::MAX as usize + 1 => { + for arg in call.args.iter() { + self.expression(arg); + } + self.emit_at( + Op::CallFnPtr { + argc: (argc - 1) as u8, + method: false, + // Call position binds no receiver at all. + receiver: None, + }, + pos, + ); + } + _ => return false, + } + true + } + + /// Whether a call can go through generic dispatch. + /// + /// Rhai resolves a handful of names syntactically in `eval_fn_call_expr` + /// before dispatch ever happens (`func/call.rs:1109-1340`), so routing + /// those through `call_fn_raw` would change what they mean. A call that + /// captures the enclosing scope is closure construction, and a qualified + /// name resolves against imported modules; neither is a plain call. + fn is_lowerable_call(&self, call: &FnCallExpr) -> bool { + // `is_shared` belongs here for a sharper reason than the rest: rhai + // answers it syntactically in both call positions (`func/call.rs:1240` + // and `:929`) and registers no function for it anywhere, so a lowered + // call raises `ErrorFunctionNotFound` where the walker returns a bool. + const SYNTACTIC: &[&str] = &["eval", "is_def_var", "is_def_fn"]; + + // These are handled by `fn_ptr_call` above, but only at the arities + // rhai treats syntactically — at any other arity it falls through to + // ordinary dispatch, and so must this. + if matches!(call.name.as_str(), "Fn" | "call" | "curry" | "is_shared") { + return false; + } + + !call.capture_parent_scope + && !call_has_namespace!(call) + && call.args.len() <= u8::MAX as usize + && !SYNTACTIC.contains(&call.name.as_str()) + } + + /// Lower `&&` or `||`: evaluate operands left to right, stopping at the + /// first that decides the result. + /// + /// Each operand is coerced to bool at its own position, which is why the + /// jumps carry one — rhai reports a non-boolean operand against the + /// operand, not the expression (`eval/expr.rs:367-399`). + fn short_circuit(&mut self, operands: &[Expr], stop_on: bool) { + let mut decided = Vec::new(); + + for operand in operands { + self.expression(operand); + let pos = operand.position(); + let site = self.code.len(); + self.emit_at( + if stop_on { + Op::JumpIfTrue { target: u32::MAX } + } else { + Op::JumpIfFalse { target: u32::MAX } + }, + pos, + ); + decided.push(site); + } + + self.emit(Op::Bool(!stop_on)); + let past = self.emit_jump(); + for site in decided { + self.patch_here(site); + } + self.emit(Op::Bool(stop_on)); + self.patch_here(past); + } + + /// Lower a block, leaving its value — the last statement's, or unit if + /// empty — on the stack, and dropping anything it declared. + fn block(&mut self, statements: &[Stmt]) -> bool { + let depth = self.slots.depth(); + + let Some((last, leading)) = statements.split_last() else { + self.emit(Op::Unit); + return true; + }; + + for stmt in leading { + if !self.statement(stmt) { + return false; + } + self.emit(Op::Pop); + } + if !self.statement(last) { + return false; + } + + self.unwind_to(depth); + true + } + + /// Lower a block for its effects only, leaving nothing on the stack. + /// + /// Loop bodies discard their value: rhai's loops yield unit or whatever a + /// `break` supplied, never the body's last statement. + fn block_discarding(&mut self, statements: &[Stmt]) -> bool { + if !self.block(statements) { + return false; + } + self.emit(Op::Pop); + true + } + + /// Emit the scope truncation for leaving a block, and unwind the + /// compile-time slot model with it. + /// + /// The value the block produced is already on the operand stack, so it + /// survives locals being dropped. + fn unwind_to(&mut self, depth: usize) { + if self.slots.depth() > depth { + let depth = u16::try_from(depth).expect("slot count is bounded"); + self.emit(Op::UnwindTo(depth)); + self.slots.unwind_to(depth as usize); + } + } + + /// Where the instruction list currently ends, for [`Lowering::rewind`]. + fn mark(&self) -> usize { + self.code.len() + } + + /// Drop everything emitted since `mark`. + /// + /// Only safe for an attempt that emitted no jumps out of the rewound + /// region, which is why it is used for chains and nothing else: a chain + /// emits its operands and then one instruction, and gives up before + /// emitting the instruction. + fn rewind(&mut self, mark: usize) { + self.code.truncate(mark); + self.positions.truncate(mark); + } + + fn here(&self) -> u32 { + u32::try_from(self.code.len()).expect("chunk length is bounded") + } + + /// Emit a jump with a placeholder target, returning its site for patching. + fn emit_jump(&mut self) -> usize { + let site = self.code.len(); + self.emit(Op::Jump(u32::MAX)); + site + } + + fn emit_jump_if_false(&mut self, pos: Position) -> usize { + let site = self.code.len(); + self.emit_at(Op::JumpIfFalse { target: u32::MAX }, pos); + site + } + + /// Point a previously emitted jump at the next instruction. + fn patch_here(&mut self, site: usize) { + let target = self.here(); + self.patch_to(site, target); + } + + /// Point a previously emitted jump at an instruction already emitted. + fn patch_to(&mut self, site: usize, target: u32) { + match &mut self.code[site] { + Op::Jump(slot) + | Op::JumpIfFalse { target: slot, .. } + | Op::JumpIfTrue { target: slot, .. } + | Op::IterNext { exit: slot, .. } + | Op::PushHandler { target: slot, .. } => *slot = target, + other => unreachable!("patched a {other:?}, which is not a jump"), + } + } + + /// Emit an `IterDrop` for every iterator live above `floor`. + fn drop_iterators(&mut self, floor: usize) { + for _ in floor..self.iters { + self.emit(Op::IterDrop); + } + } + + /// Disarm every `try` region entered above `floor`. + /// + /// A `break` or `continue` jumps over the `PopHandler` the straight-line + /// path would have run. Left armed, the handler keeps a stale target and a + /// stale set of depths, and the next error anywhere in the frame is caught + /// into a `catch` block that has already been left. + fn pop_handlers(&mut self, floor: usize) { + for _ in floor..self.handlers { + self.emit(Op::PopHandler); + } + } + + /// Open a loop whose `break` and `continue` unwind to the same place — + /// `while`, `loop` and `do`, which declare nothing of their own. + fn begin_loop(&mut self, continue_target: u32) { + let depth = u16::try_from(self.slots.depth()).expect("slot count is bounded"); + self.loops.push(Loop { + continue_target, + break_depth: depth, + continue_depth: depth, + iters: self.iters, + handlers: self.handlers, + owns_iterator: false, + breaks: Vec::new(), + }); + } + + /// Open a `for`, which does declare: the loop variable and any counter + /// live between the two depths, so leaving drops them and going round + /// again does not. + fn begin_for(&mut self, continue_target: u32, break_depth: u16) { + self.loops.push(Loop { + continue_target, + break_depth, + continue_depth: u16::try_from(self.slots.depth()).expect("slot count is bounded"), + iters: self.iters, + handlers: self.handlers, + owns_iterator: true, + breaks: Vec::new(), + }); + } + + fn end_loop(&mut self) -> Vec { + self.loops.pop().expect("loop stack is balanced").breaks + } + + fn residual_expr(&mut self, expr: &Expr) { + let residual = self.push_residual(expr.clone()); + self.emit(Op::EvalAst { + residual, + rewind_scope: true, + }); + } + + fn constant(&mut self, value: Dynamic) { + let index = self.push_const(value); + self.emit(Op::Const(index)); + } + + fn push_const(&mut self, value: Dynamic) -> u32 { + // Programs at this scale make a linear scan cheaper than a hash map, + // and it keeps the pool in emission order for readable disassembly. + let rendered = format!("{value:?}"); + if let Some(index) = self + .consts + .iter() + .position(|existing| format!("{existing:?}") == rendered) + { + return index as u32; + } + self.consts.push(value); + (self.consts.len() - 1) as u32 + } + + fn push_name(&mut self, name: ImmutableString) -> u32 { + if let Some(index) = self.names.iter().position(|existing| *existing == name) { + return index as u32; + } + self.names.push(name); + (self.names.len() - 1) as u32 + } + + /// A script uses a handful of distinct operators however many times it + /// mentions them, so the pool stays tiny and a linear scan is right. + fn push_token(&mut self, token: Token) -> u32 { + if let Some(index) = self.tokens.iter().position(|existing| *existing == token) { + return index as u32; + } + self.tokens.push(token); + (self.tokens.len() - 1) as u32 + } + + fn push_assign_op(&mut self, entry: AssignOp) -> u32 { + if let Some(index) = self + .assign_ops + .iter() + .position(|existing| *existing == entry) + { + return index as u32; + } + self.assign_ops.push(entry); + (self.assign_ops.len() - 1) as u32 + } + + fn push_residual(&mut self, expr: Expr) -> u32 { + self.residuals.push(expr); + (self.residuals.len() - 1) as u32 + } + + fn emit(&mut self, op: Op) { + // An upper bound, not the answer: no instruction pushes more than one + // value, so one slot per instruction cannot be too small. The verifier + // replaces it with the measured high water once lowering is done. + self.max_stack = self.max_stack.saturating_add(1); + self.code.push(op); + self.positions.push(Position::NONE); + } + + /// Emit an instruction that can fail against a place in the source. + /// + /// The position goes to the side table rather than into the instruction, so + /// it can be stripped from an artifact without touching the code. + fn emit_at(&mut self, op: Op, pos: Position) { + self.emit(op); + *self.positions.last_mut().expect("just emitted") = pos; + } +} + +/// One step, still as AST. +/// A step, and where rhai would blame it. +/// +/// The position travels with the step rather than being taken from the chain: +/// rhai reports each kind against its own node, and one chain instruction has +/// only one position-table entry between all of them. +enum ChainStep<'a> { + /// The index expression, and the `[` it sits behind — see [`Step::Index`]. + Index(&'a Expr, rhai::Position), + Property( + &'a ( + (ImmutableString, u64), + (ImmutableString, u64), + ImmutableString, + ), + rhai::Position, + ), + Method(&'a FnCallExpr, rhai::Position), +} + +/// Unpick rhai's nested chain encoding into a root and a list of steps. +/// +/// `a.b[i]` is `Dot { lhs: a, rhs: Index { lhs: b, rhs: i } }`: each nested +/// node's `lhs` is the current step's operand and its `rhs` is the +/// continuation, so the list is built by walking `rhs` and taking `lhs` at each +/// level. The innermost `rhs` is the last step rather than a continuation, +/// which is what ends the walk. +/// +/// `ASTFlags::BREAK` is what ends it, and it carries real information: +/// `a[b[0]]` and `a[b][0]` have the same shape, and the flag is the only thing +/// that says the first one's `b[0]` is an index expression rather than two +/// steps (`eval/chaining.rs:698`). +/// +/// Returns `None` for `?.` and `?[]`, which short-circuit on unit rather than +/// stepping, and for a dot onto anything but a property or a method. +fn flatten_chain(expr: &Expr) -> Option<(&Expr, Vec>)> { + /// A chain node's parts: operand side, continuation side, and whether the + /// step it introduces is a property rather than an index. + fn parts(expr: &Expr) -> Option<(&Expr, &Expr, ASTFlags, bool)> { + match expr { + Expr::Dot(binary, flags, ..) => Some((&binary.lhs, &binary.rhs, *flags, true)), + Expr::Index(binary, flags, ..) => Some((&binary.lhs, &binary.rhs, *flags, false)), + _ => None, + } + } + + let (root, mut rest, mut flags, mut dotted) = parts(expr)?; + let mut steps = Vec::new(); + // Rhai's `op_pos`, which is the position of the chain node the step is + // being taken *inside* rather than of the step's operand, and which walks + // down with the recursion (`eval/chaining.rs:695`). + let mut bracket = expr.position(); + + loop { + if flags.intersects(ASTFlags::NEGATED) { + return None; + } + + // `rest` is the continuation only when it is a chain node *and* this + // node is not marked as the last one. Otherwise it is this step's own + // operand — the index expression, or the property being read. + let next = (!flags.intersects(ASTFlags::BREAK)) + .then(|| parts(rest)) + .flatten(); + + let (operand, following) = match next { + Some((operand, _, _, _)) => (operand, Some(rest)), + None => (rest, None), + }; + + steps.push(match (dotted, operand) { + (true, Expr::Property(prop, pos)) => ChainStep::Property(prop, *pos), + (true, Expr::MethodCall(call, pos)) => ChainStep::Method(call, *pos), + // `a.(expr)` is not syntax, so a dot onto anything else is a shape + // the parser only makes for something handled elsewhere. + (true, _) => return None, + (false, index) => ChainStep::Index(index, bracket), + }); + + match following { + Some(node) => { + let (_, next_rest, next_flags, next_dotted) = + parts(node).expect("checked by `next`"); + rest = next_rest; + flags = next_flags; + dotted = next_dotted; + bracket = node.position(); + } + None => break, + } + } + + Some((root, steps)) +} + +/// Wrap statements as a block expression. +/// +/// `Expr::Stmt` is the one shape `eval_expression_tree_raw` routes to +/// `eval_stmt_block` rather than `eval_expr`, which is what lets statements go +/// back through the walker at all. +fn wrap_statements(statements: Vec) -> Expr { + let span = statements.first().zip(statements.last()).map_or_else( + || Span::new(Position::NONE, Position::NONE), + // `crate::types`, not `crate::types::position`: `no_position` swaps the + // module out for a zero-sized one and re-exports `Span` from whichever + // is in play. + |(first, last)| crate::types::Span::new(first.position(), last.position()), + ); + + Expr::Stmt(Box::new(StmtBlock::new_with_span(statements, span))) +} diff --git a/src/grain/compile/poolable.rs b/src/grain/compile/poolable.rs new file mode 100644 index 000000000..4e7845809 --- /dev/null +++ b/src/grain/compile/poolable.rs @@ -0,0 +1,59 @@ +use core::ops::{Range, RangeInclusive}; + +use rhai::{Array, Blob, Dynamic, Map, INT}; + +/// Whether a constant can live in the artifact's constant pool. +/// +/// Two constraints happen to coincide here, so one check enforces both. +/// +/// The artifact must be loadable in another process, which rules out anything +/// carrying a host `TypeId`, a live `Rc`, or a clock reading: `Variant`, +/// `Shared`, `TimeStamp`. +/// +/// And `FnPtr` is not a value the VM may simply clone into place, even in the +/// same process. Rhai attaches the calling environment when it reads a +/// function pointer out of a constant (`ast/expr.rs:471-482`), so a pointer +/// copied straight from the pool would be missing the module library it was +/// created against. Keeping it out of the pool leaves it as a fragment, which +/// evaluates through the path that does the attaching. +pub(crate) fn is_poolable(value: &Dynamic) -> bool { + // Under `no_float` rhai has no float type and no `is_float` to ask, so + // there is nothing here for the question to be about. + #[cfg(not(feature = "no_float"))] + if value.is_float() { + return true; + } + + if value.is_unit() || value.is_bool() || value.is_int() || value.is_char() || value.is_string() + { + return true; + } + + if value.is_array() { + return value + .read_lock::() + .map(|array| array.iter().all(is_poolable)) + .unwrap_or(false); + } + + if value.is_map() { + return value + .read_lock::() + .map(|map| map.values().all(is_poolable)) + .unwrap_or(false); + } + + if value.is_blob() { + return value.read_lock::().is_some(); + } + + // A range is a host type by representation but not by nature: rhai builds + // one for `0..5` and indexes strings and arrays with it, and its `TypeId` + // is one both sides can name. Without this every slice is a fragment. + if value.is::>() || value.is::>() { + return true; + } + + // Anything else — FnPtr, TimeStamp, Decimal, a host type, a shared cell. + false +} diff --git a/src/grain/compile/slots.rs b/src/grain/compile/slots.rs new file mode 100644 index 000000000..3dc8fe7f0 --- /dev/null +++ b/src/grain/compile/slots.rs @@ -0,0 +1,50 @@ +use rhai::ImmutableString; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +/// Assigns a slot to every local, mirroring how rhai's own `Scope` grows. +/// +/// Locals live in the caller's `Scope`, and rhai pushes an entry per +/// declaration and truncates back on block exit. So a slot is just the entry's +/// index, and a block boundary is a mark-and-truncate on this side too. +/// +/// Shadowing needs no special handling: a second `let x` pushes a second entry, +/// and resolution scans backwards, so the newer slot wins while the older one +/// stays addressable by anything compiled before it — which is exactly rhai's +/// behaviour. +#[derive(Debug, Default)] +pub(crate) struct Slots { + names: Vec, +} + +impl Slots { + /// Slot count, which is also the scope depth a block should unwind to. + pub(crate) fn depth(&self) -> usize { + self.names.len() + } + + /// Declare a local and return its slot. + pub(crate) fn declare(&mut self, name: ImmutableString) -> u16 { + let slot = self.names.len(); + self.names.push(name); + u16::try_from(slot).expect("slot count is bounded by the compiler's own limit") + } + + /// Resolve a name to the most recent slot holding it. + pub(crate) fn resolve(&self, name: &str) -> Option { + self.names + .iter() + .rposition(|candidate| candidate.as_str() == name) + .map(|slot| u16::try_from(slot).expect("slot count is bounded")) + } + + /// Drop every local declared since `depth`. + pub(crate) fn unwind_to(&mut self, depth: usize) { + self.names.truncate(depth); + } + + /// Whether another local would overflow the slot encoding. + pub(crate) fn is_full(&self) -> bool { + self.names.len() >= u16::MAX as usize + } +} diff --git a/src/grain/format/abi.rs b/src/grain/format/abi.rs new file mode 100644 index 000000000..1cdc00f4a --- /dev/null +++ b/src/grain/format/abi.rs @@ -0,0 +1,246 @@ +//! What a `Dynamic` is, on the machine that wrote the artifact. +//! +//! Rhai's feature flags change the value representation rather than just what +//! is available: `f32_float` makes `FLOAT` an `f32`, `only_i32` narrows `INT`, +//! `sync` swaps `Rc` for `Arc`. Loading an artifact across one of those is not +//! a missing feature, it is a value decoded as the wrong type — so the header +//! carries a fingerprint and the loader refuses a mismatch by name. +//! +//! ## What the fingerprint can and cannot see +//! +//! Widths are *measured*, so they are right no matter how rhai was configured. +//! The booleans are read from this crate's own features, which is why the +//! manifest mirrors them — `cfg!(feature = "no_object")` here does not consult +//! rhai's manifest. +//! +//! That leaves one gap: enabling a restriction on rhai directly, bypassing the +//! mirror. The cross-checks below close it wherever rust can prove the +//! disagreement, turning it into a compile error rather than a wrong +//! fingerprint. They cannot close it everywhere, which is what the mirror is +//! documented for. + +/// Restrictions that are not visible in a width. +/// +/// Order is the wire order and must never change; append only. A flag's name +/// is what the loader reports, so it has to match rhai's own spelling. +const FLAGS: &[(&str, bool)] = &[ + ("sync", cfg!(feature = "sync")), + ("decimal", cfg!(feature = "decimal")), + ("no_index", cfg!(feature = "no_index")), + ("no_object", cfg!(feature = "no_object")), + ("no_closure", cfg!(feature = "no_closure")), + ("no_function", cfg!(feature = "no_function")), + ("no_module", cfg!(feature = "no_module")), + ("no_position", cfg!(feature = "no_position")), + ("no_custom_syntax", cfg!(feature = "no_custom_syntax")), + ("no_time", cfg!(feature = "no_time")), + ("unchecked", cfg!(feature = "unchecked")), +]; + +/// `Engine` is only `Send + Sync` when rhai is built with `sync`, so claiming +/// the flag without rhai agreeing fails to compile. +#[cfg(feature = "sync")] +const _: () = { + const fn assert_sync() {} + let _ = assert_sync::; +}; + +/// The value representation an artifact was written against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Abi { + /// `size_of::()`. Measured, so `only_i32` set on rhai alone is + /// still caught. + pub int_bytes: u8, + /// `size_of::()`, or 0 under `no_float`. + pub float_bytes: u8, + /// `FLAGS` as a bitmask, low bit first. + pub flags: u32, +} + +/// How two fingerprints differ. +/// +/// Naming the difference is the whole point: "artifact was built for a +/// different rhai" is not something a user can act on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AbiMismatch { + /// A width differs, which means integers or floats would decode wrong. + Width { + /// Which width differs + what: &'static str, + /// What the writer used + artifact: u8, + /// What this build uses + host: u8, + }, + /// A restriction differs. `artifact` is whether the writer had it on. + Flag { + /// Which flag differs + flag: &'static str, + /// Whether the writer had it on + artifact: bool, + }, +} + +impl core::fmt::Display for AbiMismatch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Width { + what, + artifact, + host, + } => write!( + f, + "artifact was written with a {artifact}-byte {what}, but this build has {host}" + ), + Self::Flag { flag, artifact } => { + let (writer, reader) = if *artifact { + ("on", "off") + } else { + ("off", "on") + }; + write!( + f, + "artifact was written with `{flag}` {writer}, but this build has it {reader}" + ) + } + } + } +} + +impl Abi { + /// The fingerprint of the running build. + #[must_use] + pub fn host() -> Self { + #[cfg(not(feature = "no_float"))] + let float_bytes = core::mem::size_of::() as u8; + #[cfg(feature = "no_float")] + let float_bytes = 0u8; + + let mut flags = 0u32; + for (bit, (_, on)) in FLAGS.iter().enumerate() { + if *on { + flags |= 1 << bit; + } + } + + Self { + int_bytes: core::mem::size_of::() as u8, + float_bytes, + flags, + } + } + + /// Why this fingerprint cannot be loaded by `host`, if it cannot. + /// + /// Widths first: they are measured rather than declared, so they are the + /// claim least likely to be lying. + #[must_use] + pub fn incompatible_with(self, host: Self) -> Option { + if self.int_bytes != host.int_bytes { + return Some(AbiMismatch::Width { + what: "INT", + artifact: self.int_bytes, + host: host.int_bytes, + }); + } + if self.float_bytes != host.float_bytes { + return Some(AbiMismatch::Width { + what: "FLOAT", + artifact: self.float_bytes, + host: host.float_bytes, + }); + } + + let differing = self.flags ^ host.flags; + if differing != 0 { + let bit = differing.trailing_zeros() as usize; + // A bit past the table means the writer knew a flag this build does + // not. Reporting it as unknown beats indexing out of bounds. + let flag = FLAGS + .get(bit) + .map_or("an unknown restriction", |(name, _)| *name); + return Some(AbiMismatch::Flag { + flag, + artifact: self.flags & (1 << bit) != 0, + }); + } + + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_build_can_load_its_own_artifacts() { + assert_eq!(Abi::host().incompatible_with(Abi::host()), None); + } + + #[test] + fn the_build_measures_the_widths_it_should() { + let abi = Abi::host(); + // Against the aliases rather than concrete types, so this holds under + // only_i32 and f32_float as well — the point is that the fingerprint + // reports the build it was taken on, whichever build that is. + assert_eq!(abi.int_bytes as usize, core::mem::size_of::(),); + #[cfg(not(feature = "no_float"))] + assert_eq!( + abi.float_bytes as usize, + core::mem::size_of::(), + ); + #[cfg(feature = "no_float")] + assert_eq!(abi.float_bytes, 0); + } + + #[test] + fn a_narrower_int_is_refused_by_name() { + let host = Abi::host(); + // Halved rather than named: `only_i32` makes 4 the host's own width, + // and an artifact agreeing with the host is not a mismatch to report. + let narrow = Abi { + int_bytes: host.int_bytes / 2, + ..host + }; + assert_eq!( + narrow.incompatible_with(host), + Some(AbiMismatch::Width { + what: "INT", + artifact: host.int_bytes / 2, + host: host.int_bytes, + }), + ); + } + + /// The message has to name the flag; that is the difference between an + /// error a user can act on and one they cannot. + #[test] + fn a_differing_restriction_is_refused_by_name() { + let host = Abi::host(); + let restricted = Abi { + flags: host.flags ^ (1 << 3), + ..host + }; + + let Some(mismatch @ AbiMismatch::Flag { flag, .. }) = restricted.incompatible_with(host) + else { + panic!("a differing flag must be refused"); + }; + assert_eq!(flag, "no_object"); + assert!(mismatch.to_string().contains("no_object")); + } + + #[test] + fn a_flag_this_build_has_never_heard_of_does_not_panic() { + let host = Abi::host(); + let future = Abi { + flags: host.flags ^ (1 << 31), + ..host + }; + assert!(matches!( + future.incompatible_with(host), + Some(AbiMismatch::Flag { .. }), + )); + } +} diff --git a/src/grain/format/mod.rs b/src/grain/format/mod.rs new file mode 100644 index 000000000..aaff1ec3c --- /dev/null +++ b/src/grain/format/mod.rs @@ -0,0 +1,347 @@ +//! The on-the-wire form of a [`Program`]. +//! +//! This is what the project is for. A device that loads bytes runs no parser +//! and builds no tree, so neither the nodes a retained rhai `AST` costs nor the +//! parser's higher peak is ever spent. Everything else — the speed, the +//! verifier — is downstream of being able to write a program out and read it +//! back somewhere else. +//! +//! ## Shape +//! +//! ```text +//! "RGRN" magic +//! u16 format version +//! abi INT width, FLOAT width, restriction bitmask +//! varint + utf8 source name, empty for none +//! section names +//! section constants +//! section operator tokens +//! section op-assignments +//! section chains +//! section switch tables, prefixed with a hasher probe +//! varint declared max stack +//! section code, verbatim +//! section position table, empty when stripped +//! ``` +//! +//! Everything outside the code section is LEB128, signed values zigzagged, +//! because it is read once at load. The code section is not: it is the bytes +//! the VM executes, copied in and sliced back out untouched, with fixed-width +//! operands so dispatch does not decode. See [`crate::grain::bytecode::code`]. +//! +//! ## What it refuses to write +//! +//! Residual fragments are real `Expr` trees — precisely the allocation this +//! removes — so a program holding any is rejected rather than partially +//! written. A script function the compiler could not lower is rejected for the +//! same reason: rhai keeps its own copy of that one, as an AST. Both failures +//! name what blocked them, because "cannot serialize" without the construct is +//! not something a script author can act on. + +use core::convert::TryFrom; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +mod abi; +mod read; +mod write; + +pub use abi::{Abi, AbiMismatch}; +pub use read::ReadError; +pub use write::WriteError; + +use crate::grain::bytecode::VerifyError; +use crate::grain::program::Program; + +/// Identifies the format, so a file that is not one fails immediately rather +/// than as a nonsense opcode. +const MAGIC: [u8; 4] = *b"RGRN"; + +/// Bumped when an encoding changes in a way an older reader would misread. +/// Additive changes that an older reader would reject anyway — a new op tag, +/// a new constant tag — do not need it. +const VERSION: u16 = 7; + +/// Where a chain starts. Append only. +mod root_tag { + pub const LOCAL: u8 = 0x01; + pub const TEMPORARY: u8 = 0x02; + pub const NAMED: u8 = 0x03; + pub const THIS: u8 = 0x04; +} + +/// Chain-step tags. Append only. +mod step_tag { + pub const INDEX: u8 = 0x01; + pub const PROPERTY: u8 = 0x02; + pub const METHOD: u8 = 0x03; +} + +/// What a chain does at the end. Append only. +mod tail_tag { + pub const READ: u8 = 0x01; + pub const ASSIGN: u8 = 0x02; + pub const ASSIGN_OP: u8 = 0x03; +} + +/// Constant-pool tags, over the subset of `Dynamic` that means the same thing +/// in another process. Append only. +/// +/// The whole table is defined on every build even where a restriction feature +/// means nothing can produce a given tag — `no_float` cannot write a `FLOAT`, +/// `no_index` an `ARRAY`. The numbering is the wire format, so it must not +/// shift with the features of whoever compiled the writer. +#[allow(dead_code)] +mod constant { + pub const UNIT: u8 = 0x00; + pub const FALSE: u8 = 0x01; + pub const TRUE: u8 = 0x02; + pub const INT: u8 = 0x03; + pub const FLOAT: u8 = 0x04; + pub const CHAR: u8 = 0x05; + pub const STRING: u8 = 0x06; + pub const ARRAY: u8 = 0x07; + pub const MAP: u8 = 0x08; + pub const BLOB: u8 = 0x09; + pub const RANGE: u8 = 0x0a; + pub const RANGE_INCLUSIVE: u8 = 0x0b; +} + +impl<'a> Program<'a> { + /// Encode this program, diagnostics included. + /// + /// # Errors + /// + /// Fails if the program still holds anything that cannot cross a process + /// boundary: an un-lowered fragment, a script function, or a constant + /// carrying a host type. + pub fn write(&self) -> Result, WriteError> { + write::write(self, write::Positions::Keep) + } + + /// Encode this program without its position table, returning the table + /// separately. + /// + /// This is the split the debug layer exists for. Ship the first half to the + /// device and keep the second: errors then arrive carrying an instruction + /// address, and [`pos::resolve`](crate::grain::pos::resolve) turns it back into a position + /// where the source still is. The table can also be sent back later with + /// [`Program::attach_positions`]. + /// + /// # Errors + /// + /// As [`Program::write`]. + pub fn write_stripped(&self) -> Result<(Vec, Vec), WriteError> { + let bytes = write::write(self, write::Positions::Strip)?; + Ok((bytes, self.positions().to_table())) + } + + /// Decode a program written by [`Program::write`], borrowing its + /// instructions from `bytes`. + /// + /// Nothing is allocated for the code — the returned program points into the + /// buffer and the VM dispatches on it where it lies. What is allocated is + /// bounded by the distinct constants, names and operators the script + /// mentions, not by how long it is. Call [`Program::into_owned`] if the + /// buffer has to go. + /// + /// The chunk is verified before this returns, so a program that loads + /// cannot underflow the operand stack, jump outside itself, jump into the + /// middle of an instruction, or index a pool entry that is not there. With + /// the code being executed in place, that check is the only thing between a + /// corrupt file and an operand read as an opcode. + /// + /// # Errors + /// + /// Fails on a bad header, an ABI the running build cannot represent, + /// truncated or malformed input, or a chunk that does not verify. + pub fn read(bytes: &'a [u8]) -> Result { + read::read(bytes) + } +} + +/// A reader positioned in a byte slice. +struct Cursor<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, pos: 0 } + } + + fn take(&mut self, n: usize) -> Result<&'a [u8], ReadError> { + let end = self.pos.checked_add(n).ok_or(ReadError::Truncated)?; + let slice = self.bytes.get(self.pos..end).ok_or(ReadError::Truncated)?; + self.pos = end; + Ok(slice) + } + + fn byte(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + /// A count that has to be reserved for before it is read. + /// + /// Nothing is encoded in less than a byte, so a count larger than what is + /// left cannot be honest — and reserving for it first would let a handful + /// of bytes ask for a terabyte. Reading the entries one at a time runs out + /// of input safely; `Vec::with_capacity` does not, because it allocates + /// before the first entry is read. Found by `tests/fuzz.rs`. + fn count(&mut self) -> Result { + let count = usize::try_from(self.uvarint()?).map_err(|_| ReadError::Truncated)?; + if count > self.bytes.len() - self.pos { + return Err(ReadError::Truncated); + } + Ok(count) + } + + /// LEB128, capped at ten groups so a run of continuation bytes cannot spin. + fn uvarint(&mut self) -> Result { + let mut value = 0u64; + for shift in (0..64).step_by(7) { + let byte = self.byte()?; + let payload = u64::from(byte & 0x7f); + // The tenth group has a single bit left to land in. Shifting would + // drop the other six rather than refuse them, so a value too wide + // for 64 bits would decode as a smaller one. + if shift == 63 && payload > 1 { + return Err(ReadError::MalformedVarint); + } + value |= payload << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + } + Err(ReadError::MalformedVarint) + } + + fn ivarint(&mut self) -> Result { + let raw = self.uvarint()?; + Ok(((raw >> 1) as i64) ^ -((raw & 1) as i64)) + } + + fn index(&mut self) -> Result { + u32::try_from(self.uvarint()?).map_err(|_| ReadError::MalformedVarint) + } + + fn small(&mut self) -> Result { + u16::try_from(self.uvarint()?).map_err(|_| ReadError::MalformedVarint) + } + + fn str(&mut self) -> Result<&'a str, ReadError> { + let len = usize::try_from(self.uvarint()?).map_err(|_| ReadError::Truncated)?; + core::str::from_utf8(self.take(len)?).map_err(|_| ReadError::BadUtf8) + } + + fn at_end(&self) -> bool { + self.pos == self.bytes.len() + } +} + +fn put_uvarint(out: &mut Vec, mut value: u64) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +fn put_ivarint(out: &mut Vec, value: i64) { + put_uvarint(out, ((value << 1) ^ (value >> 63)) as u64); +} + +fn put_str(out: &mut Vec, value: &str) { + put_uvarint(out, value.len() as u64); + out.extend_from_slice(value.as_bytes()); +} + +impl From for ReadError { + fn from(err: VerifyError) -> Self { + Self::Unverifiable(err) + } +} + +impl From for ReadError { + fn from(err: crate::grain::bytecode::BadTable) -> Self { + Self::Names(err) + } +} + +impl From for ReadError { + fn from(err: crate::grain::bytecode::TableError) -> Self { + Self::Positions(err) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Varints are the whole encoding's foundation; a rounding error here + /// misreads every index in the file. + #[test] + fn unsigned_varints_round_trip_at_the_edges() { + for value in [0u64, 1, 127, 128, 300, u32::MAX as u64, u64::MAX] { + let mut buf = Vec::new(); + put_uvarint(&mut buf, value); + assert_eq!(Cursor::new(&buf).uvarint().unwrap(), value, "at {value}"); + } + } + + #[test] + fn signed_varints_round_trip_across_zero() { + for value in [0i64, -1, 1, -64, 63, i32::MIN as i64, i64::MIN, i64::MAX] { + let mut buf = Vec::new(); + put_ivarint(&mut buf, value); + assert_eq!(Cursor::new(&buf).ivarint().unwrap(), value, "at {value}"); + } + } + + /// Small numbers are most of a chunk, so the encoding only pays for itself + /// if they cost one byte. + #[test] + fn small_indices_cost_one_byte() { + let mut buf = Vec::new(); + put_uvarint(&mut buf, 127); + assert_eq!(buf.len(), 1); + } + + #[test] + fn a_run_of_continuation_bytes_terminates() { + let never_ends = vec![0xffu8; 64]; + assert_eq!( + Cursor::new(&never_ends).uvarint(), + Err(ReadError::MalformedVarint), + ); + } + + /// The tenth group is the one place a shift could silently lose bits, so a + /// wide value there must be refused rather than truncated into a small one. + #[test] + fn a_tenth_group_wider_than_one_bit_is_refused() { + let mut ten = [0x80u8; 10]; + + // The largest value there is: nine full groups and a final bit. + let widest = [[0xffu8; 9].as_slice(), &[0x01]].concat(); + assert_eq!(Cursor::new(&widest).uvarint(), Ok(u64::MAX)); + + // One past it. Shifting would drop the payload and read this as zero. + ten[9] = 0x02; + assert_eq!(Cursor::new(&ten).uvarint(), Err(ReadError::MalformedVarint)); + + ten[9] = 0x7f; + assert_eq!(Cursor::new(&ten).uvarint(), Err(ReadError::MalformedVarint)); + } + + #[test] + fn reading_past_the_end_is_an_error_not_a_panic() { + assert_eq!(Cursor::new(&[]).byte(), Err(ReadError::Truncated)); + assert_eq!(Cursor::new(&[1, 2]).take(9), Err(ReadError::Truncated)); + } +} diff --git a/src/grain/format/read.rs b/src/grain/format/read.rs new file mode 100644 index 000000000..68044c233 --- /dev/null +++ b/src/grain/format/read.rs @@ -0,0 +1,488 @@ +use crate::{tokenizer::Token, Dynamic, ImmutableString}; +use core::convert::{TryFrom, TryInto}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +use crate::grain::bytecode::{ + AssignOp, BadTable, Chain, Chunk, Positions, Root, Step, Strings, Switch, SwitchCase, + SwitchRange, TableError, Tail, VerifyError, +}; +use crate::grain::format::abi::{Abi, AbiMismatch}; +use crate::grain::format::{constant, root_tag, step_tag, tail_tag, Cursor, MAGIC, VERSION}; +use crate::grain::program::{Function, Parts, Program}; + +/// How deeply a constant may nest. +/// +/// Decoding an array or a map recurses, so an artifact claiming a few thousand +/// nested arrays would overflow the stack of whatever loads it. Rhai's own +/// parser caps expression depth for the same reason; this is the loader's +/// version, and it is well past anything a literal in real source reaches. +const MAX_CONSTANT_DEPTH: usize = 64; + +/// Why an artifact could not be loaded. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadError { + /// Not a rhaigrain artifact at all. + BadMagic, + /// Written by a format this build does not know how to read. + UnsupportedVersion { + /// The version the artifact claims + found: u16, + /// The version this build reads + supported: u16, + }, + /// Written against a different value representation. Loading anyway would + /// decode integers or floats as the wrong type. + Abi(AbiMismatch), + /// The input ended mid-value. + Truncated, + /// A varint that never terminates, or one too wide for its field. + MalformedVarint, + /// A string that is not UTF-8. + BadUtf8, + /// A tag this build has no meaning for. + UnknownTag { + /// Which section it was read from + section: &'static str, + /// The tag itself + tag: u8, + }, + /// An operator syntax rhai does not recognise. + UnknownToken { + /// The syntax that was read + syntax: String, + }, + /// The artifact's `switch` case hashes were computed by a differently + /// seeded hasher, so none of them would ever match. + HashSeedMismatch { + /// The seed the writer used + artifact: u64, + /// The seed this build uses + host: u64, + }, + /// Constants nested past `MAX_CONSTANT_DEPTH`. + ConstantTooDeep, + /// Bytes left over after the last section, so the file is not what it + /// claims to be even though every field parsed. + TrailingBytes { + /// How many bytes are left over + count: usize, + }, + /// The chunk parsed but does not agree with itself. + Unverifiable(VerifyError), + /// The position table is malformed, or belongs to a different program. + Positions(TableError), + /// The name table's spans do not fit its blob. + Names(BadTable), +} + +impl core::fmt::Display for ReadError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::BadMagic => f.write_str("not a rhaigrain artifact"), + Self::UnsupportedVersion { found, supported } => write!( + f, + "artifact is format version {found}, and this build reads {supported}" + ), + Self::Abi(mismatch) => write!(f, "{mismatch}"), + Self::Truncated => f.write_str("artifact ends mid-value"), + Self::MalformedVarint => f.write_str("malformed varint"), + Self::BadUtf8 => f.write_str("a string is not valid UTF-8"), + Self::UnknownTag { section, tag } => { + write!(f, "unknown {section} tag {tag:#04x}") + } + Self::UnknownToken { syntax } => write!(f, "`{syntax}` is not an operator"), + Self::HashSeedMismatch { artifact, host } => write!( + f, + "this artifact's `switch` cases were hashed with a different seed \ + ({artifact:#018x} against {host:#018x}), so none of them could match — \ + call `rhai::config::hashing::set_hashing_seed` with the same seed \ + wherever this was compiled and wherever it is loaded" + ), + Self::ConstantTooDeep => write!( + f, + "a constant nests deeper than {MAX_CONSTANT_DEPTH} levels" + ), + Self::TrailingBytes { count } => { + write!(f, "{count} byte(s) follow the last section") + } + Self::Unverifiable(err) => write!(f, "chunk failed verification: {err:?}"), + Self::Positions(err) => write!(f, "{err}"), + Self::Names(err) => write!(f, "name table is malformed: {err:?}"), + } + } +} + +pub(super) fn read(bytes: &[u8]) -> Result, ReadError> { + let mut cursor = Cursor::new(bytes); + + if cursor.take(MAGIC.len())? != MAGIC { + return Err(ReadError::BadMagic); + } + + let version = u16::from_le_bytes(cursor.take(2)?.try_into().expect("two bytes")); + if version != VERSION { + return Err(ReadError::UnsupportedVersion { + found: version, + supported: VERSION, + }); + } + + // Before anything is decoded: past here every value is read as a type the + // fingerprint just promised. + let abi = Abi { + int_bytes: cursor.byte()?, + float_bytes: cursor.byte()?, + flags: u32::from_le_bytes(cursor.take(4)?.try_into().expect("four bytes")), + }; + if let Some(mismatch) = abi.incompatible_with(Abi::host()) { + return Err(ReadError::Abi(mismatch)); + } + + let source = cursor.str()?; + let source = (!source.is_empty()).then(|| ImmutableString::from(source)); + + // Borrowed: the spans are read, the blob is sliced, and nothing per-name + // is allocated. + let count = cursor.count()?; + let mut starts = Vec::with_capacity(count + 1); + starts.push(0u32); + for _ in 0..count { + starts.push(cursor.index()?); + } + let blob_len = usize::try_from(cursor.uvarint()?).map_err(|_| ReadError::Truncated)?; + let names = Strings::borrowed(cursor.take(blob_len)?, starts)?; + + let mut consts = Vec::new(); + for _ in 0..cursor.uvarint()? { + consts.push(get_constant(&mut cursor, 0)?); + } + + let mut tokens = Vec::new(); + for _ in 0..cursor.uvarint()? { + tokens.push(get_token(&mut cursor)?); + } + + let mut assign_ops = Vec::new(); + for _ in 0..cursor.uvarint()? { + assign_ops.push(AssignOp { + op_assign: get_token(&mut cursor)?, + op_assign_name: cursor.index()?, + op: get_token(&mut cursor)?, + op_name: cursor.index()?, + }); + } + + let mut chains = Vec::new(); + for _ in 0..cursor.uvarint()? { + chains.push(get_chain(&mut cursor)?); + } + + let switches = get_switches(&mut cursor)?; + + let main = get_chunk(&mut cursor)?; + + let mut functions = Vec::new(); + for _ in 0..cursor.uvarint()? { + let name = cursor.index()?; + // Zero is "untyped"; anything else is an index one higher. + let this_type = match cursor.uvarint()? { + 0 => None, + raw => Some(u32::try_from(raw - 1).map_err(|_| ReadError::Truncated)?), + }; + let mut params = Vec::new(); + for _ in 0..cursor.uvarint()? { + params.push(cursor.index()?); + } + functions.push(Function { + name, + this_type, + params, + // Not encoded: derived from the chunk by `Program::new`, so a loaded + // program and a compiled one cannot disagree about it. + takes_this: false, + chunk: get_chunk(&mut cursor)?, + }); + } + + // Borrowed, not copied. The VM dispatches on these bytes where they lie. + let code_len = usize::try_from(cursor.uvarint()?).map_err(|_| ReadError::Truncated)?; + let code = cursor.take(code_len)?; + + // Absent means stripped, which is the normal shape for something that + // reached a device. `from_table` refuses a table belonging to another + // program, so a mismatched pair fails here rather than misreporting later. + let table_len = usize::try_from(cursor.uvarint()?).map_err(|_| ReadError::Truncated)?; + let positions = if table_len == 0 { + Positions::Stripped + } else { + Positions::from_table(cursor.take(table_len)?, code_len)? + }; + + if !cursor.at_end() { + return Err(ReadError::TrailingBytes { + count: bytes.len() - cursor.pos, + }); + } + + let program = Program::new( + code.into(), + main, + functions, + Parts { + positions, + residuals: Vec::new(), + consts, + names, + tokens, + assign_ops, + chains, + switches, + // Script functions are still ASTs, so `write` refuses a program + // that has any and a loaded one never does. + lib: None, + #[cfg(not(feature = "no_module"))] + resolver: None, + source, + }, + ); + + // An artifact is untrusted input, so the chunk is checked before it is + // handed to a `Vm`. Nothing that fails here is constructible by the + // compiler — and because the VM executes these bytes in place, this is what + // stands between a corrupt file and an operand read as an opcode. + program.verify()?; + + Ok(program) +} + +/// A chain step's own position. Line zero means none. +fn get_position(cursor: &mut Cursor) -> Result { + let line = cursor.small()?; + let column = cursor.small()?; + Ok(if line == 0 { + rhai::Position::NONE + } else { + rhai::Position::new(line, column) + }) +} + +fn get_chain(cursor: &mut Cursor) -> Result { + let root = match cursor.byte()? { + root_tag::LOCAL => Root::Local { + slot: cursor.small()?, + name: cursor.index()?, + }, + root_tag::NAMED => Root::Named { + name: cursor.index()?, + pos: get_position(cursor)?, + }, + root_tag::THIS => Root::This { + pos: get_position(cursor)?, + }, + root_tag::TEMPORARY => Root::Temporary, + tag => { + return Err(ReadError::UnknownTag { + section: "chain root", + tag, + }) + } + }; + let operands = cursor.small()?; + + let mut steps = Vec::new(); + for _ in 0..cursor.uvarint()? { + steps.push(match cursor.byte()? { + step_tag::INDEX => Step::Index { + operand: cursor.small()?, + pos: get_position(cursor)?, + bracket: get_position(cursor)?, + }, + step_tag::PROPERTY => Step::Property { + name: cursor.index()?, + getter: cursor.index()?, + setter: cursor.index()?, + pos: get_position(cursor)?, + }, + step_tag::METHOD => Step::Method { + name: cursor.index()?, + argc: cursor.byte()?, + operand: cursor.small()?, + pos: get_position(cursor)?, + }, + tag => { + return Err(ReadError::UnknownTag { + section: "chain step", + tag, + }) + } + }); + } + + let tail = match cursor.byte()? { + tail_tag::READ => Tail::Read, + tail_tag::ASSIGN => Tail::Assign { op: None }, + tail_tag::ASSIGN_OP => Tail::Assign { + op: Some(cursor.index()?), + }, + tag => { + return Err(ReadError::UnknownTag { + section: "chain tail", + tag, + }) + } + }; + + Ok(Chain { + root, + steps, + tail, + operands, + }) +} + +/// Read the switch tables, refusing them if their case hashes were made by a +/// hasher this process cannot reproduce. +/// +/// The check is not belt and braces: without it a seed mismatch loads cleanly +/// and every `switch` silently takes its default, which is a wrong answer +/// rather than a failure. See `write::put_switches`. +fn get_switches(cursor: &mut Cursor) -> Result, ReadError> { + let count = cursor.uvarint()?; + if count == 0 { + return Ok(Vec::new()); + } + + let artifact = u64::from_le_bytes(cursor.take(8)?.try_into().expect("eight bytes")); + let host = crate::grain::bytecode::probe(); + if artifact != host { + return Err(ReadError::HashSeedMismatch { artifact, host }); + } + + let mut switches = Vec::new(); + for _ in 0..count { + let mut cases = Vec::new(); + for _ in 0..cursor.uvarint()? { + cases.push(SwitchCase { + hash: u64::from_le_bytes(cursor.take(8)?.try_into().expect("eight bytes")), + target: cursor.index()?, + }); + } + + let mut ranges = Vec::new(); + for _ in 0..cursor.uvarint()? { + ranges.push(SwitchRange { + from: bounded_int(cursor.ivarint()?)?, + to: bounded_int(cursor.ivarint()?)?, + inclusive: cursor.byte()? != 0, + target: cursor.index()?, + }); + } + + switches.push(Switch { + cases, + ranges, + default: cursor.index()?, + }); + } + Ok(switches) +} + +/// Narrow a written bound back to this build's `INT`. +/// +/// The ABI fingerprint has already promised the widths agree, so this can only +/// fail on a corrupt file — but a range bound is compared against a subject, +/// and a silently truncated one would match the wrong values. +fn bounded_int(value: i64) -> Result { + rhai::INT::try_from(value).map_err(|_| ReadError::MalformedVarint) +} + +/// Read a chunk's span. The verifier is what checks it names real code. +fn get_chunk(cursor: &mut Cursor) -> Result { + let entry = cursor.index()?; + let end = cursor.index()?; + Ok(Chunk::new(entry, end, cursor.small()?)) +} + +fn get_token(cursor: &mut Cursor) -> Result { + let syntax = cursor.str()?; + Token::lookup_symbol_from_syntax(syntax).ok_or_else(|| ReadError::UnknownToken { + syntax: syntax.to_string(), + }) +} + +fn get_constant(cursor: &mut Cursor, depth: usize) -> Result { + if depth > MAX_CONSTANT_DEPTH { + return Err(ReadError::ConstantTooDeep); + } + + Ok(match cursor.byte()? { + constant::UNIT => Dynamic::UNIT, + constant::FALSE => Dynamic::from(false), + constant::TRUE => Dynamic::from(true), + + constant::INT => { + let value = cursor.ivarint()?; + Dynamic::from(rhai::INT::try_from(value).map_err(|_| ReadError::MalformedVarint)?) + } + + #[cfg(not(feature = "no_float"))] + constant::FLOAT => { + let width = core::mem::size_of::(); + let bits = cursor.take(width)?; + Dynamic::from(rhai::FLOAT::from_le_bytes( + bits.try_into().expect("width matches the fingerprint"), + )) + } + + constant::CHAR => { + let code = cursor.index()?; + Dynamic::from(char::from_u32(code).ok_or(ReadError::MalformedVarint)?) + } + + constant::STRING => Dynamic::from(ImmutableString::from(cursor.str()?)), + + constant::ARRAY => { + // The declared length is untrusted, so nothing is reserved from it; + // a short file runs out of bytes instead of out of memory. + let count = cursor.uvarint()?; + let mut array = rhai::Array::new(); + for _ in 0..count { + array.push(get_constant(cursor, depth + 1)?); + } + Dynamic::from(array) + } + + constant::MAP => { + let count = cursor.uvarint()?; + let mut map = rhai::Map::new(); + for _ in 0..count { + let key = cursor.str()?.into(); + map.insert(key, get_constant(cursor, depth + 1)?); + } + Dynamic::from(map) + } + + constant::RANGE => { + let start = bounded_int(cursor.ivarint()?)?; + Dynamic::from(start..bounded_int(cursor.ivarint()?)?) + } + + constant::RANGE_INCLUSIVE => { + let start = bounded_int(cursor.ivarint()?)?; + Dynamic::from(start..=bounded_int(cursor.ivarint()?)?) + } + + constant::BLOB => { + let len = usize::try_from(cursor.uvarint()?).map_err(|_| ReadError::Truncated)?; + Dynamic::from(cursor.take(len)?.to_vec()) + } + + tag => { + return Err(ReadError::UnknownTag { + section: "constant", + tag, + }) + } + }) +} diff --git a/src/grain/format/write.rs b/src/grain/format/write.rs new file mode 100644 index 000000000..41762ac6a --- /dev/null +++ b/src/grain/format/write.rs @@ -0,0 +1,448 @@ +use core::ops::{Range, RangeInclusive}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +use rhai::{tokenizer::Token, Array, Blob, Dynamic, Map, INT}; + +use crate::grain::bytecode::{AssignOp, Chain, Root, Step, Tail}; +use crate::grain::format::abi::Abi; +use crate::grain::format::{ + constant, put_ivarint, put_str, put_uvarint, root_tag, step_tag, tail_tag, MAGIC, VERSION, +}; +use crate::grain::program::Program; + +/// Why a program cannot be written out. +/// +/// Every variant names the construct that blocked it. A serializer that only +/// says "no" leaves the author guessing which line to change. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteError { + /// The program still hands fragments to rhai's walker, and a fragment is a + /// real `Expr` tree. + /// + /// Names the first construct responsible and where it is, because a caller + /// deciding whether to ship source instead needs to know what it is + /// falling back for. + HasResiduals { + /// How many fragments the program still has + count: usize, + /// What the first of them is + construct: &'static str, + /// Where it is in the source + pos: rhai::Position, + }, + /// The program still carries rhai's own function library rather than + /// chunks, so its functions are ASTs an artifact cannot hold. + HasScriptFunctions, + /// A pooled constant carries something that has no meaning in another + /// process — a host type, a function pointer, a clock reading. + UnserializableConstant { + /// Index into the constant pool + index: usize, + /// What the constant holds + type_name: String, + }, + /// An operator token that does not survive `syntax -> token`. Storing it + /// would silently change which built-in the VM reaches. + AmbiguousToken { + /// The token's syntax + token: String, + }, +} + +impl core::fmt::Display for WriteError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::HasResiduals { + count, + construct, + pos, + } => write!( + f, + "{construct} at {pos} is not compiled yet, so this program still has \ + {count} fragment(s) that only rhai's walker can evaluate" + ), + Self::HasScriptFunctions => { + f.write_str("script functions are still ASTs and cannot be written") + } + Self::UnserializableConstant { index, type_name } => write!( + f, + "constant {index} is a `{type_name}`, which has no meaning in another process" + ), + Self::AmbiguousToken { token } => { + write!(f, "operator token `{token}` does not survive a round trip") + } + } + } +} + +/// Whether an artifact carries its own diagnostics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Positions { + Keep, + Strip, +} + +pub(super) fn write(program: &Program, positions: Positions) -> Result, WriteError> { + // Refuse before encoding anything, so a rejection cannot leave a caller + // holding a half-written buffer that happens to parse. + if program.residual_count() > 0 { + let (construct, pos) = program + .first_unsupported() + .unwrap_or(("an unlowered expression", rhai::Position::NONE)); + return Err(WriteError::HasResiduals { + count: program.residual_count(), + construct, + pos, + }); + } + if program.lib().is_some() { + return Err(WriteError::HasScriptFunctions); + } + + let mut out = Vec::new(); + out.extend_from_slice(&MAGIC); + out.extend_from_slice(&VERSION.to_le_bytes()); + + let abi = Abi::host(); + out.push(abi.int_bytes); + out.push(abi.float_bytes); + out.extend_from_slice(&abi.flags.to_le_bytes()); + + put_str(&mut out, program.source().map_or("", |s| s.as_str())); + + // One blob and a list of spans, so a loader can point at the names where + // they lie instead of allocating one box per name. + let names = program.names(); + put_uvarint(&mut out, names.len() as u64); + for start in names.starts().iter().skip(1) { + put_uvarint(&mut out, u64::from(*start)); + } + put_uvarint(&mut out, names.blob().len() as u64); + out.extend_from_slice(names.blob()); + + put_uvarint(&mut out, program.consts().len() as u64); + for (index, value) in program.consts().iter().enumerate() { + put_constant(&mut out, value) + .map_err(|type_name| WriteError::UnserializableConstant { index, type_name })?; + } + + put_uvarint(&mut out, program.tokens().len() as u64); + for token in program.tokens() { + put_token(&mut out, token)?; + } + + put_uvarint(&mut out, program.assign_ops().len() as u64); + for entry in program.assign_ops() { + put_assign_op(&mut out, entry)?; + } + + put_uvarint(&mut out, program.chains().len() as u64); + for chain in program.chains() { + put_chain_spec(&mut out, chain); + } + + put_switches(&mut out, program.switches()); + + // Chunks: main first, then one per compiled function. Entry offsets are + // into the single code buffer below. + put_chunk(&mut out, program.main()); + put_uvarint(&mut out, program.functions().len() as u64); + for function in program.functions() { + put_uvarint(&mut out, u64::from(function.name)); + // Zero is "untyped", so an index arrives one higher. The field is why + // `VERSION` moved to 7: it sits inside a positional record, and a reader + // that did not expect it would take it for the parameter count and lose + // its place in every section that follows. + put_uvarint(&mut out, function.this_type.map_or(0, |t| u64::from(t) + 1)); + put_uvarint(&mut out, function.params.len() as u64); + for param in &function.params { + put_uvarint(&mut out, u64::from(*param)); + } + put_chunk(&mut out, &function.chunk); + } + + // Verbatim. This is the whole point of the byte encoding: what a loader + // hands the VM is a slice of the artifact, not something rebuilt from it. + let code = program.code(); + put_uvarint(&mut out, code.len() as u64); + out.extend_from_slice(code); + + // Last, and length-prefixed, so removing it is a truncation rather than a + // re-encode — and so a reader that finds nothing there is reading a + // deliberately stripped artifact, not a damaged one. + let table = match positions { + Positions::Keep => program.positions().to_table(), + Positions::Strip => Vec::new(), + }; + put_uvarint(&mut out, table.len() as u64); + out.extend_from_slice(&table); + + Ok(out) +} + +/// A step's position, which travels with the step rather than in the position +/// table — see [`Step::pos`]. Line zero means none, because rhai's own line +/// numbers start at one. +fn put_position(out: &mut Vec, pos: rhai::Position) { + put_uvarint(out, pos.line().unwrap_or(0) as u64); + put_uvarint(out, pos.position().unwrap_or(0) as u64); +} + +fn put_chain_spec(out: &mut Vec, chain: &Chain) { + match chain.root { + Root::Local { slot, name } => { + out.push(root_tag::LOCAL); + put_uvarint(out, u64::from(slot)); + put_uvarint(out, u64::from(name)); + } + Root::Named { name, pos } => { + out.push(root_tag::NAMED); + put_uvarint(out, u64::from(name)); + put_position(out, pos); + } + Root::This { pos } => { + out.push(root_tag::THIS); + put_position(out, pos); + } + Root::Temporary => out.push(root_tag::TEMPORARY), + } + put_uvarint(out, u64::from(chain.operands)); + + put_uvarint(out, chain.steps.len() as u64); + for step in &chain.steps { + match step { + Step::Index { + operand, + pos, + bracket, + } => { + out.push(step_tag::INDEX); + put_uvarint(out, u64::from(*operand)); + put_position(out, *pos); + put_position(out, *bracket); + } + Step::Property { + name, + getter, + setter, + pos, + } => { + out.push(step_tag::PROPERTY); + put_uvarint(out, u64::from(*name)); + put_uvarint(out, u64::from(*getter)); + put_uvarint(out, u64::from(*setter)); + put_position(out, *pos); + } + Step::Method { + name, + argc, + operand, + pos, + } => { + out.push(step_tag::METHOD); + put_uvarint(out, u64::from(*name)); + out.push(*argc); + put_uvarint(out, u64::from(*operand)); + put_position(out, *pos); + } + } + } + + match &chain.tail { + Tail::Read => out.push(tail_tag::READ), + Tail::Assign { op: None } => out.push(tail_tag::ASSIGN), + Tail::Assign { op: Some(op) } => { + out.push(tail_tag::ASSIGN_OP); + put_uvarint(out, u64::from(*op)); + } + } +} + +/// Write the switch tables, behind a probe that says whether their hashes will +/// still mean anything on the other side. +/// +/// Rhai's parser keeps only the *hash* of a case value (`ast/stmt.rs:336`), so +/// there is nothing here to re-hash at load — and by default those hashes do +/// not survive the trip, because rhai's default features include +/// `ahash/runtime-rng` and the seed is drawn per process. An artifact with a +/// `switch` in it therefore requires `config::hashing::set_hashing_seed` with +/// the same seed on both sides. +/// +/// The probe is what turns that from a silent wrong answer — every subject +/// dispatched to the default — into a refusal to load. It goes here rather +/// than in the ABI fingerprint because it only constrains artifacts that +/// actually contain a `switch`; making every program agree about a hashing +/// seed would be a restriction bought for nothing. +fn put_switches(out: &mut Vec, switches: &[crate::grain::bytecode::Switch]) { + put_uvarint(out, switches.len() as u64); + if switches.is_empty() { + return; + } + out.extend_from_slice(&crate::grain::bytecode::probe().to_le_bytes()); + + for switch in switches { + put_uvarint(out, switch.cases.len() as u64); + for case in &switch.cases { + // Fixed width: a hash is eight bytes of noise, which a varint + // would spend ten on. + out.extend_from_slice(&case.hash.to_le_bytes()); + put_uvarint(out, u64::from(case.target)); + } + + put_uvarint(out, switch.ranges.len() as u64); + for range in &switch.ranges { + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(range.from)); + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(range.to)); + out.push(u8::from(range.inclusive)); + put_uvarint(out, u64::from(range.target)); + } + + put_uvarint(out, u64::from(switch.default)); + } +} + +fn put_chunk(out: &mut Vec, chunk: &crate::grain::bytecode::Chunk) { + put_uvarint(out, u64::from(chunk.entry())); + put_uvarint(out, u64::from(chunk.end())); + put_uvarint(out, u64::from(chunk.max_stack())); +} + +/// Store an operator token as its syntax, and prove that reading it back gives +/// the same token. +/// +/// The check is not ceremony. `Plus` and `UnaryPlus` share the syntax `"+"`, +/// so the reverse lookup collapses them — and the token is what the built-in +/// operator lookup keys on, so a collapsed one would quietly reach a different +/// implementation. +fn put_token(out: &mut Vec, token: &Token) -> Result<(), WriteError> { + let ambiguous = || WriteError::AmbiguousToken { + token: format!("{token:?}"), + }; + + if !token.is_literal() { + return Err(ambiguous()); + } + let syntax = token.literal_syntax(); + if Token::lookup_symbol_from_syntax(syntax).as_ref() != Some(token) { + return Err(WriteError::AmbiguousToken { + token: syntax.to_string(), + }); + } + put_str(out, syntax); + Ok(()) +} + +fn put_assign_op(out: &mut Vec, entry: &AssignOp) -> Result<(), WriteError> { + put_token(out, &entry.op_assign)?; + put_uvarint(out, u64::from(entry.op_assign_name)); + put_token(out, &entry.op)?; + put_uvarint(out, u64::from(entry.op_name)); + Ok(()) +} + +/// Encode a constant, or name the type that stopped it. +/// +/// The accepted set is `compile::poolable::is_poolable`'s, which the compiler +/// already applies when filling the pool — so a rejection here means the two +/// have drifted apart, not that a script did something exotic. +fn put_constant(out: &mut Vec, value: &Dynamic) -> Result<(), String> { + if value.is_unit() { + out.push(constant::UNIT); + return Ok(()); + } + if let Ok(flag) = value.as_bool() { + out.push(if flag { + constant::TRUE + } else { + constant::FALSE + }); + return Ok(()); + } + if let Ok(number) = value.as_int() { + out.push(constant::INT); + // `INT` narrows to `i32` under `only_i32`, so the widening is real + // there even though it is a no-op here. The fingerprint is what stops + // the reader from decoding the wrong width back. + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(number)); + return Ok(()); + } + #[cfg(not(feature = "no_float"))] + if let Ok(number) = value.as_float() { + out.push(constant::FLOAT); + out.extend_from_slice(&number.to_le_bytes()); + return Ok(()); + } + if let Ok(character) = value.as_char() { + out.push(constant::CHAR); + put_uvarint(out, u32::from(character).into()); + return Ok(()); + } + if value.is_string() { + let text = value + .read_lock::() + .ok_or_else(|| value.type_name().to_string())?; + out.push(constant::STRING); + put_str(out, text.as_str()); + return Ok(()); + } + if value.is_array() { + let array = value + .read_lock::() + .ok_or_else(|| value.type_name().to_string())?; + out.push(constant::ARRAY); + put_uvarint(out, array.len() as u64); + for item in array.iter() { + put_constant(out, item)?; + } + return Ok(()); + } + if value.is_map() { + let map = value + .read_lock::() + .ok_or_else(|| value.type_name().to_string())?; + out.push(constant::MAP); + put_uvarint(out, map.len() as u64); + for (key, item) in map.iter() { + put_str(out, key.as_str()); + put_constant(out, item)?; + } + return Ok(()); + } + if value.is_blob() { + let blob = value + .read_lock::() + .ok_or_else(|| value.type_name().to_string())?; + out.push(constant::BLOB); + put_uvarint(out, blob.len() as u64); + out.extend_from_slice(&blob); + return Ok(()); + } + + // Behind the array and map checks because those are far more common, and + // after everything cheap because reaching one means two failed downcasts. + if let Some(range) = value.read_lock::>() { + out.push(constant::RANGE); + put_range(out, range.start, range.end); + return Ok(()); + } + if let Some(range) = value.read_lock::>() { + out.push(constant::RANGE_INCLUSIVE); + put_range(out, *range.start(), *range.end()); + return Ok(()); + } + + Err(value.type_name().to_string()) +} + +/// `INT` widens to `i64` on the wire; the fingerprint is what stops a reader +/// narrowing it back to something else. +fn put_range(out: &mut Vec, start: INT, end: INT) { + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(start)); + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(end)); +} diff --git a/src/grain/mod.rs b/src/grain/mod.rs new file mode 100644 index 000000000..04c6c864f --- /dev/null +++ b/src/grain/mod.rs @@ -0,0 +1,99 @@ +//! A bytecode VM for Rhai. +//! +//! Rhai evaluates by walking its AST, which the parser allocates a node at a +//! time — so holding a script costs in proportion to how much program it is, +//! and the parser's peak is higher again than what it settles at. That is what +//! caps script size on a small target long before anything else does. +//! rhaigrain compiles the tree to a flat instruction stream that can be +//! produced elsewhere and loaded without a parser. +//! +//! `tests/grain/allocation.rs` measures both ends of that with a tracking +//! allocator. +//! +//! Execution reuses the host `Engine`: `Dynamic` stays the value type and every +//! registered function is dispatched by rhai itself. Only control flow, local +//! variable access and operator fast paths are reimplemented. +//! +//! A program that has been lowered all the way through can be written out with +//! [`Program::write`] and read back with [`Program::read`] — see [`mod@format`]. +//! That is the artifact the device loads, and the reason the tree never has to +//! exist there. +//! +//! Coverage is total from the start, by construction rather than by effort. +//! Anything the compiler cannot yet lower is kept as an AST fragment and handed +//! back to rhai's walker through [`bytecode::Op::EvalAst`], so a `Program` +//! always means the same thing as the `AST` it came from. Progress is measured +//! by [`Program::residual_count`] falling, not by constructs becoming legal. +//! +//! # Compiling and running +//! +//! The `Engine` does the parsing and, at runtime, all the dispatching; the VM +//! only replaces the walk between those two. +//! +//! ``` +//! use rhai::grain::{Compiler, Vm}; +//! use rhai::{Engine, Scope}; +//! +//! let engine = Engine::new(); +//! let ast = engine.compile("let total = 0; for i in 0..10 { total += i; } total")?; +//! +//! let program = Compiler::new().compile(&ast); +//! +//! // The `Scope` is the caller's locals: a script declares are left in it, +//! // exactly as `Engine::eval_with_scope` would. +//! let mut scope = Scope::new(); +//! let value = Vm::new(&engine).eval_with_scope(&mut scope, &program)?; +//! +//! assert_eq!(value.as_int().unwrap(), 45); +//! # Ok::<_, Box>(()) +//! ``` +//! +//! # Shipping an artifact +//! +//! The point of the byte encoding: compile on a host, run somewhere that never +//! sees the source. A loaded `Program` borrows its instructions from the bytes, +//! so nothing it retains grows with how long the script is. +//! +//! ``` +//! use rhai::grain::{Compiler, Program, Vm}; +//! use rhai::{Engine, Scope}; +//! +//! let engine = Engine::new(); +//! +//! // On the host. +//! let ast = engine.compile("let x = 6; x * 7")?; +//! let program = Compiler::new().compile(&ast); +//! +//! // `write` refuses a program still holding AST fragments, so this is also +//! // the check that the script lowered all the way through. +//! assert_eq!(program.residual_count(), 0); +//! let bytes = program.write().expect("no residuals, so it is writable"); +//! +//! // On the device, with no parser and no `AST` in sight. +//! let loaded = Program::read(&bytes).expect("written by this build"); +//! let value = Vm::new(&engine).eval(&loaded)?; +//! +//! assert_eq!(value.as_int().unwrap(), 42); +//! # Ok::<_, Box>(()) +//! ``` +//! +//! Diagnostics are separable: [`Program::write_stripped`] hands back the +//! artifact and its position table separately, so the device carries only the +//! first and a failure there comes back as an instruction address the host +//! turns into a source position with [`pos::resolve`]. + +// A VM that runs untrusted bytecode has no business containing any, and saying +// so here makes it the compiler's problem rather than a promise. `crates/ +// rhaigrain-pos` declares the same. +#![forbid(unsafe_code)] + +pub mod bytecode; +mod compile; +pub mod format; +pub mod pos; +mod program; +mod vm; + +pub use compile::Compiler; +pub use program::Program; +pub use vm::Vm; diff --git a/src/grain/pos/mod.rs b/src/grain/pos/mod.rs new file mode 100644 index 000000000..02455d084 --- /dev/null +++ b/src/grain/pos/mod.rs @@ -0,0 +1,327 @@ +//! Turning a rhaigrain instruction address back into a source position, +//! without `std`. +//! +//! A rhaigrain artifact carries no positions in its instruction stream. They +//! live in a separate table, so a device can be shipped the bytecode alone and +//! the table kept on the host that compiled it — which is the point, because a +//! position on every instruction is a large fraction of the artifact and is +//! never read unless something fails. +//! +//! What comes back is a [`Site`]: a line and a column as plain numbers. This +//! crate does not know what a `rhai::Position` is, and deliberately so, since +//! the reason to keep the table on-device is to report an error without +//! linking the compiler that produced it. +//! +//! ## Where this sits +//! +//! It resolves the first hop only. A script that was minified before it was +//! compiled needs a second: the [`Site`] is a position in the *minified* +//! source, and a Source Map v3 resolver such as `rhaiper-map` takes it the rest +//! of the way to the original. +//! +//! ```text +//! instruction address --[this crate]--> minified line:col --[a source map]--> original +//! ``` +//! +//! ## Table format +//! +//! ```text +//! varint entry count +//! per entry: varint address delta from the previous entry +//! varint line, 1-based (0 means the site is unknown) +//! varint column, 1-based (0 means the start of a line) +//! ``` +//! +//! Entries are sorted by address, so a lookup walks until it reaches or passes +//! the one it wants and stops. That is a linear scan, which is the right shape +//! for something only consulted when a program has already failed. + +pub mod varint; + +/// A place in a source file. +/// +/// Both fields follow rhai's own convention: `line` counts from 1, and +/// `column` counts characters from 1 with 0 meaning the start of a line. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Site { + /// 1-based line. + pub line: u32, + /// 1-based character column; 0 is the start of a line. + pub column: u32, +} + +/// Why a table could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + /// The table ends mid-entry. + Truncated, + /// A number too wide for the field it was read into. + Overflow, + /// Addresses are not strictly ascending, so the scan's early exit would be + /// wrong and a lookup could silently return the wrong site. + OutOfOrder { + /// The address that did not ascend + at: u32, + }, + /// Bytes remain after the last entry, so this is not the table it claims. + TrailingBytes { + /// How many bytes are left over + count: usize, + }, +} + +impl From for Error { + fn from(err: varint::Error) -> Self { + match err { + varint::Error::Truncated => Self::Truncated, + varint::Error::Overflow => Self::Overflow, + } + } +} + +/// The site recorded for instruction `address`, if the table has one. +/// +/// `None` means the instruction has no recorded site — not that the table is +/// broken. Most instructions have none: only those that can raise an error +/// against a place in the source are worth recording. +/// +/// A malformed table also reads as `None`. Resolving happens while reporting +/// an error, and failing to resolve must not replace the error being reported; +/// call [`check`] at load time to find out whether a table is sound. +#[must_use] +pub fn resolve(table: &[u8], address: u32) -> Option { + let mut at = 0usize; + let count = varint::u32(table, &mut at).ok()?; + + let mut current = 0u32; + for _ in 0..count { + current = current.checked_add(varint::u32(table, &mut at).ok()?)?; + let line = varint::u32(table, &mut at).ok()?; + let column = varint::u32(table, &mut at).ok()?; + + if current == address { + return (line != 0).then_some(Site { line, column }); + } + // Ascending, so nothing past here can match. + if current > address { + return None; + } + } + + None +} + +/// How many entries the table holds, without decoding them. +/// +/// # Errors +/// +/// [`Error::Truncated`] or [`Error::Overflow`] if the count itself is unreadable. +pub fn count(table: &[u8]) -> Result { + let mut at = 0usize; + Ok(varint::u32(table, &mut at)?) +} + +/// Check that a table is well formed, so [`resolve`] can afford not to. +/// +/// # Errors +/// +/// Names the first thing wrong: a truncated entry, a number too wide, an +/// address that does not ascend, or bytes past the end. +pub fn check(table: &[u8]) -> Result<(), Error> { + let mut at = 0usize; + let count = varint::u32(table, &mut at)?; + + let mut previous: Option = None; + for _ in 0..count { + let delta = varint::u32(table, &mut at)?; + let address = match previous { + None => delta, + // A zero delta repeats the previous address, which would shadow it. + Some(previous) if delta == 0 => return Err(Error::OutOfOrder { at: previous }), + Some(previous) => previous.checked_add(delta).ok_or(Error::Overflow)?, + }; + let _line = varint::u32(table, &mut at)?; + let _column = varint::u32(table, &mut at)?; + previous = Some(address); + } + + if at != table.len() { + return Err(Error::TrailingBytes { + count: table.len() - at, + }); + } + + Ok(()) +} + +/// Build a table from sites in ascending address order. +/// +/// Entries with a zero line are dropped: they carry no information, and +/// keeping them would cost bytes on a table whose whole purpose is to be small +/// enough to leave behind. +/// +/// # Panics +/// +/// Panics if the addresses are not strictly ascending. That is a caller bug — +/// a compiler emits in address order by construction — and producing a table +/// that silently resolves wrong is worse than not producing one. +#[must_use] +pub fn encode(sites: impl IntoIterator) -> alloc::vec::Vec { + let kept: alloc::vec::Vec<_> = sites + .into_iter() + .filter(|(_, site)| site.line != 0) + .collect(); + + let mut out = alloc::vec::Vec::new(); + varint::put_u64(&mut out, kept.len() as u64); + + let mut previous = 0u32; + for (index, (address, site)) in kept.iter().enumerate() { + if index > 0 { + assert!( + *address > previous, + "table addresses must strictly ascend: {address} follows {previous}", + ); + } + varint::put_u64(&mut out, u64::from(address - previous)); + varint::put_u64(&mut out, u64::from(site.line)); + varint::put_u64(&mut out, u64::from(site.column)); + previous = *address; + } + + out +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use alloc::vec::Vec; + + fn site(line: u32, column: u32) -> Site { + Site { line, column } + } + + fn table() -> Vec { + encode(vec![ + (0, site(1, 5)), + (3, site(1, 11)), + (40, site(7, 2)), + (41, site(9, 0)), + ]) + } + + #[test] + fn a_recorded_address_resolves_to_its_site() { + let table = table(); + assert_eq!(resolve(&table, 0), Some(site(1, 5))); + assert_eq!(resolve(&table, 3), Some(site(1, 11))); + assert_eq!(resolve(&table, 40), Some(site(7, 2))); + } + + /// Column 0 is the start of a line, which is a real position and must not + /// be confused with an absent one. + #[test] + fn the_start_of_a_line_is_a_position() { + assert_eq!(resolve(&table(), 41), Some(site(9, 0))); + } + + #[test] + fn an_address_with_no_site_resolves_to_nothing() { + let table = table(); + assert_eq!(resolve(&table, 1), None, "between two entries"); + assert_eq!(resolve(&table, 99), None, "past the last entry"); + } + + #[test] + fn an_empty_table_resolves_nothing_and_is_sound() { + let table = encode(Vec::new()); + assert_eq!(check(&table), Ok(())); + assert_eq!(count(&table), Ok(0)); + assert_eq!(resolve(&table, 0), None); + } + + /// A site with no line carries nothing, and the table exists to be small. + #[test] + fn sites_with_no_line_are_not_stored() { + let table = encode(vec![(0, site(1, 1)), (1, site(0, 0)), (2, site(3, 3))]); + assert_eq!(count(&table), Ok(2)); + assert_eq!(resolve(&table, 1), None); + assert_eq!(resolve(&table, 2), Some(site(3, 3))); + } + + /// Deltas are the reason the table is worth encoding rather than storing + /// flat: consecutive instructions cost one byte of address each. + #[test] + fn a_dense_run_costs_three_bytes_an_entry() { + let dense: Vec<_> = (0..100).map(|pc| (pc, site(1, 1))).collect(); + let table = encode(dense); + assert_eq!(table.len(), 1 + 100 * 3); + } + + #[test] + fn a_sound_table_passes_its_own_check() { + assert_eq!(check(&table()), Ok(())); + } + + #[test] + fn a_truncated_table_is_named_as_such() { + let table = table(); + for cut in 1..table.len() { + assert!( + check(&table[..cut]).is_err(), + "a {cut}-byte prefix passed the check", + ); + } + } + + #[test] + fn a_repeated_address_is_refused() { + // Hand-built, since `encode` will not produce one. + let mut bytes = Vec::new(); + varint::put_u64(&mut bytes, 2); + for _ in 0..2 { + varint::put_u64(&mut bytes, 0); // delta + varint::put_u64(&mut bytes, 1); // line + varint::put_u64(&mut bytes, 1); // column + } + assert_eq!(check(&bytes), Err(Error::OutOfOrder { at: 0 })); + } + + #[test] + fn bytes_past_the_last_entry_are_refused() { + let mut bytes = table(); + bytes.push(0); + assert_eq!(check(&bytes), Err(Error::TrailingBytes { count: 1 })); + } + + /// Resolution runs while an error is being reported. Whatever the table + /// says, it may not become the failure. + #[test] + fn no_byte_string_can_make_resolution_panic() { + let table = table(); + for index in 0..table.len() { + for bit in 0..8 { + let mut corrupt = table.clone(); + corrupt[index] ^= 1 << bit; + for address in 0..64 { + let _ = resolve(&corrupt, address); + } + let _ = check(&corrupt); + } + } + + for junk in [&b""[..], &[0xff][..], &[0xff; 32][..], &[0x80; 12][..]] { + for address in 0..8 { + let _ = resolve(junk, address); + } + let _ = check(junk); + } + } + + #[test] + #[should_panic(expected = "must strictly ascend")] + fn encoding_out_of_order_sites_is_a_caller_bug() { + let _ = encode(vec![(5, site(1, 1)), (2, site(1, 1))]); + } +} diff --git a/src/grain/pos/varint.rs b/src/grain/pos/varint.rs new file mode 100644 index 000000000..a2133ece7 --- /dev/null +++ b/src/grain/pos/varint.rs @@ -0,0 +1,170 @@ +//! LEB128, the encoding everything in a rhaigrain artifact is counted in. +//! +//! Small numbers dominate a chunk — pool indices, jump targets, line numbers — +//! and almost all of them fit in one byte. Signed values are zigzagged first, +//! so a small negative costs no more than a small positive. +//! +//! Decoding takes the cursor by reference rather than owning one, so the same +//! functions serve a reader that tracks its own errors and a resolver that has +//! nothing but a byte slice. + +/// Why a varint could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + /// The bytes ran out mid-value. + Truncated, + /// A value wider than 64 bits, or a run of continuation bytes that never + /// ends. Refusing both is what stops a corrupt table from spinning. + Overflow, +} + +/// Read an unsigned varint, advancing `at`. +/// +/// # Errors +/// +/// [`Error::Truncated`] if the slice ends first, [`Error::Overflow`] if the +/// value does not fit in 64 bits. +pub fn u64(bytes: &[u8], at: &mut usize) -> Result { + let mut value = 0u64; + for shift in (0..64).step_by(7) { + let byte = *bytes.get(*at).ok_or(Error::Truncated)?; + *at += 1; + let payload = u64::from(byte & 0x7f); + // The tenth group has a single bit left to land in. Shifting would drop + // the other six rather than refuse them, so a value too wide for 64 bits + // would decode as a smaller one. + if shift == 63 && payload > 1 { + return Err(Error::Overflow); + } + value |= payload << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + } + Err(Error::Overflow) +} + +/// Read a zigzagged signed varint, advancing `at`. +/// +/// # Errors +/// +/// As [`u64`](fn@u64). +pub fn i64(bytes: &[u8], at: &mut usize) -> Result { + let raw = u64(bytes, at)?; + Ok(((raw >> 1) as i64) ^ -((raw & 1) as i64)) +} + +/// Read an unsigned varint that must fit a `u32`, advancing `at`. +/// +/// # Errors +/// +/// As [`u64`](fn@u64), plus [`Error::Overflow`] if the value is too wide for +/// the field. +pub fn u32(bytes: &[u8], at: &mut usize) -> Result { + core::convert::TryFrom::try_from(u64(bytes, at)?).map_err(|_| Error::Overflow) +} + +/// Append an unsigned varint. +pub fn put_u64(out: &mut alloc::vec::Vec, mut value: u64) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +/// Append a zigzagged signed varint. +pub fn put_i64(out: &mut alloc::vec::Vec, value: i64) { + put_u64(out, ((value << 1) ^ (value >> 63)) as u64); +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec::Vec; + + #[test] + fn unsigned_values_round_trip_at_the_edges() { + for value in [0u64, 1, 127, 128, 16383, 16384, u32::MAX as u64, u64::MAX] { + let mut buf = Vec::new(); + put_u64(&mut buf, value); + let mut at = 0; + assert_eq!(u64(&buf, &mut at), Ok(value), "at {value}"); + assert_eq!( + at, + buf.len(), + "the reader must consume exactly what was written" + ); + } + } + + #[test] + fn signed_values_round_trip_across_zero() { + for value in [0i64, -1, 1, -64, 63, i32::MIN as i64, i64::MIN, i64::MAX] { + let mut buf = Vec::new(); + put_i64(&mut buf, value); + let mut at = 0; + assert_eq!(i64(&buf, &mut at), Ok(value), "at {value}"); + } + } + + /// The encoding only pays for itself if the common case is one byte. + #[test] + fn values_under_128_cost_one_byte() { + let mut buf = Vec::new(); + put_u64(&mut buf, 127); + assert_eq!(buf.len(), 1); + put_i64(&mut buf, -64); + assert_eq!(buf.len(), 2); + } + + #[test] + fn a_run_of_continuation_bytes_terminates() { + let mut at = 0; + assert_eq!(u64(&[0xff; 64], &mut at), Err(Error::Overflow)); + } + + #[test] + fn running_out_of_bytes_is_an_error_not_a_panic() { + let mut at = 0; + assert_eq!(u64(&[], &mut at), Err(Error::Truncated)); + let mut at = 0; + assert_eq!(u64(&[0x80], &mut at), Err(Error::Truncated)); + } + + /// The tenth group is the one place a shift could silently lose bits, so a + /// wide value there must be refused rather than truncated into a small one. + #[test] + fn a_tenth_group_wider_than_one_bit_is_refused() { + let mut ten = [0x80u8; 10]; + + // The largest value there is: nine full groups and a final bit. + ten[9] = 0x01; + let mut at = 0; + assert_eq!( + u64(&[[0xffu8; 9].as_slice(), &[0x01]].concat(), &mut at), + Ok(u64::MAX) + ); + + // One past it. Shifting would drop the payload and read this as zero. + ten[9] = 0x02; + let mut at = 0; + assert_eq!(u64(&ten, &mut at), Err(Error::Overflow)); + + ten[9] = 0x7f; + let mut at = 0; + assert_eq!(u64(&ten, &mut at), Err(Error::Overflow)); + } + + #[test] + fn a_value_too_wide_for_its_field_is_refused() { + let mut buf = Vec::new(); + put_u64(&mut buf, u64::from(u32::MAX) + 1); + let mut at = 0; + assert_eq!(u32(&buf, &mut at), Err(Error::Overflow)); + } +} diff --git a/src/grain/program.rs b/src/grain/program.rs new file mode 100644 index 000000000..be26ac161 --- /dev/null +++ b/src/grain/program.rs @@ -0,0 +1,795 @@ +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +use crate::ast::{ASTFlags, ASTNode}; +#[cfg(not(feature = "no_module"))] +use crate::module_resolvers::StaticModuleResolver; +use crate::{ast::Expr, ast::Stmt, tokenizer::Token, Dynamic, ImmutableString, Module, Shared}; + +use crate::grain::bytecode::{ + AssignOp, Chain, Chunk, Code, Op, Pools, Positions, Root, Strings, Switch, TableError, +}; + +/// rhai's own `SharedModule`, which it does not re-export. +pub(crate) type SharedModule = Shared; + +/// A program a native function can be handed a way back into. +/// +/// [`Vm::eval_with_callbacks`](crate::grain::Vm::eval_with_callbacks) registers one +/// wrapper per compiled function, and rhai requires a registered function to be +/// `'static` — so the program cannot still be borrowing an artifact, and the +/// wrappers have to share ownership of it rather than borrow it. +pub type SharedProgram = Shared>; + +/// One compiled script function. +/// +/// Called by [`Op::Call`](crate::bytecode::Op::Call) directly, without going +/// through rhai's dispatch: the name is already an index into the same pool the +/// call site used, so matching one is two integer comparisons rather than a +/// hash and a module walk. +#[derive(Debug, Clone)] +pub struct Function { + /// Index into the name pool. + pub name: u32, + /// Parameter names, in order, as name-pool indices. They become the + /// callee's first locals, which is what makes them slot 0 upwards. + pub params: Vec, + /// The receiver type this function was declared for, as a name-pool index. + /// + /// `fn .name()`. Rhai folds it into the function's hash rather than + /// checking it (`func/hashing.rs:159`), and tries the typed hash before the + /// plain one on a method call (`func/call.rs:614-629`) — so a typed function + /// and an untyped one of the same name and arity can both exist, and which + /// runs depends on the receiver's runtime type name. The string is what the + /// parser interned, which is already `Engine::map_type_name`'s answer. + /// + /// `None` for an ordinary function, which is nearly all of them. + pub this_type: Option, + /// Whether the body reads or writes the frame's receiver. + /// + /// Derived from the chunk rather than encoded — see [`takes_this`]. It is + /// what keeps such a function reachable by rhai, which needs the body to + /// size a call a native makes through a pointer. + pub takes_this: bool, + pub chunk: Chunk, +} + +/// A compiled script, ready to run against an `Engine`. +/// +/// Owns everything execution needs that is not the `Engine` itself, so the +/// original `AST` can be dropped after compiling. On a small target that is the +/// whole point: the tree is the part whose cost scales with the program. +/// +/// The lifetime is the artifact's. A program read from bytes borrows its +/// instructions from them and allocates only its pools, which are bounded by +/// the distinct constants and names a script actually mentions rather than by +/// how long it is. [`Program::into_owned`] cuts the tie when that is wanted. +/// +/// `residuals` is the exception, and the reason a `Program` is not always +/// serializable. Fragments rhai's walker still has to evaluate are held as real +/// `Expr` trees, which is precisely the allocation we are trying to remove. The +/// artifact format refuses to write a `Program` that has any, so nothing +/// reaching a device can depend on them. +pub struct Program<'a> { + /// Every chunk's instructions, concatenated: main first, then each + /// function. One buffer means one position table and one instruction + /// address, so a device that fails reports a single number. + code: Code<'a>, + + main: Chunk, + + /// Script functions compiled to chunks. Empty when none were compiled, + /// which is when rhai's own versions are carried in `lib` instead. + functions: Vec, + + /// The deepest chunk's operand-stack need, cached. + max_stack: u16, + + /// Whether the program can hand a function pointer to something that + /// might call it back. See [`Program::makes_fn_pointers`]. + makes_fn_pointers: bool, + + /// Whether any function was declared for a receiver type. + /// + /// Derived, not stored: [`Program::method`] is a linear scan on every method + /// call, and typed-first selection would double it for the overwhelming + /// majority of programs that have nothing typed to find. + has_typed_methods: bool, + + /// Where each instruction came from, or [`Positions::Stripped`]. + /// + /// Separable on purpose: a device is shipped the code and the host keeps + /// the table, so an error arrives as an instruction address and is resolved + /// where the source is. See [`crate::bytecode::Positions`]. + positions: Positions, + + residuals: Vec, + + /// Values `Op::Const` indexes. Deduplicated, so a constant repeated across + /// the script is stored once. + consts: Vec, + + /// Every name the program mentions, as one borrowed blob. + /// + /// Nothing needs a `String` of its own. Call names, operators, getters and + /// property keys go to rhai as `&str`; a `Scope` entry name goes in as an + /// `Identifier`, which is a `SmartString` and keeps a short name inline + /// rather than on the heap. So the whole table is two allocations — the + /// blob and the spans — and neither grows with how many names there are. + names: Strings<'a>, + + /// Operator tokens the built-in lookup keys on. A `Token` does not fit an + /// operand, and one script uses a handful of distinct operators however + /// many times it mentions them. + tokens: Vec, + + /// What each `x op= y` site needs, for the same reason. + assign_ops: Vec, + + /// The steps of each `a.b[i].c`. Out of the instruction stream because a + /// chain is one instruction however many steps it has. + chains: Vec, + + /// One dispatch table per `switch`, for the same reason. + /// + /// Case hashes are rhai's, and rhai's hasher is seeded per process unless + /// the host says otherwise — so an artifact carrying any of these carries + /// a [`probe`](crate::bytecode::probe) too, and refuses to load against a + /// hasher that would disagree with it. + switches: Vec, + + /// Script functions the compiler did not lower, as rhai's own library, so + /// a fragment can still call one the ordinary way. + /// + /// `None` when the script declared none, which is every program that came + /// from an artifact. An empty `Module` is 264 bytes and a reference count, + /// which is a fifth of what loading a small program retains — worth not + /// allocating for something nothing will look in. + lib: Option, + + /// Reinstated on the runtime state at each run, mirroring + /// `Engine::eval_ast_with_scope_raw`, so `import` resolves as it would have. + #[cfg(not(feature = "no_module"))] + resolver: Option>, + + /// Names the script in error messages and `NativeCallContext::call_source`. + source: Option, +} + +/// A summary rather than a dump: the library alone would render every script +/// function's whole AST, which is never what someone printing a `Program` +/// wants to read. +impl core::fmt::Debug for Program<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Program") + .field("source", &self.source) + .field("bytes", &self.code.len()) + .field("max_stack", &self.main.max_stack()) + .field("consts", &self.consts.len()) + .field("names", &self.names.len()) + .field("residuals", &self.residuals.len()) + .field("compiled_fns", &self.functions.len()) + .field( + "walked_fns", + &self.lib.as_ref().map_or(0, |lib| lib.count().1), + ) + .field("positions", &!self.positions.is_stripped()) + .finish() + } +} + +/// What a script author would call the construct at this node, for the +/// constructs the compiler does not lower yet. +/// +/// Only the ones worth naming: an author can act on "switch at line 42", not +/// on "Expr::Dot". Anything else falls through to the generic message. +fn unsupported_kind(node: &ASTNode) -> Option<&'static str> { + Some(match node { + ASTNode::Stmt(stmt) => match stmt { + Stmt::Switch(..) => "switch", + Stmt::For(..) => "for", + Stmt::TryCatch(..) => "try/catch", + #[cfg(not(feature = "no_module"))] + Stmt::Import(..) => "import", + #[cfg(not(feature = "no_module"))] + Stmt::Export(..) => "export", + #[cfg(not(feature = "no_closure"))] + Stmt::Share(..) => "a closure capture", + Stmt::Return(_, flags, ..) if flags.intersects(ASTFlags::BREAK) => "throw", + _ => return None, + }, + ASTNode::Expr(expr) => match expr { + Expr::InterpolatedString(..) => "string interpolation", + #[cfg(not(feature = "no_custom_syntax"))] + Expr::Custom(..) => "custom syntax", + Expr::Map(..) => "a non-constant map literal", + _ => return None, + }, + }) +} + +fn node_position(node: &ASTNode) -> rhai::Position { + match node { + ASTNode::Stmt(stmt) => stmt.position(), + ASTNode::Expr(expr) => expr.start_position(), + } +} + +/// Whether any instruction in `code` produces a function pointer. +/// +/// Read off the bytes rather than tracked while lowering, because a program +/// read from an artifact has no lowering to have tracked it — and the answer +/// has to be the same either way or one of the two paths silently loses its +/// callbacks. +/// +/// Deliberately over-broad: it says yes to a pointer that is only ever called +/// directly. Narrowing it would mean deciding where a pointer *goes*, which is +/// a dataflow question over values that outlive the instruction that made +/// them. Being wrong the other way loses a call at run time. +pub(crate) fn makes_fn_pointers(code: &[u8]) -> bool { + use crate::grain::bytecode::code::tag; + + crate::grain::bytecode::disassemble(code) + .any(|(at, ..)| matches!(code[at], tag::MAKE_CLOSURE | tag::MAKE_FN_PTR | tag::CURRY)) +} + +/// Whether a chunk reads or writes the frame's receiver. +/// +/// Read off the bytes for [`makes_fn_pointers`]'s reason: a program loaded from +/// an artifact never saw the body it came from. +/// +/// It decides whether rhai has to be able to reach the function itself. A +/// pointer to a `this`-taking chunk cannot go through a native wrapper — the +/// wrapper is registered at one arity, and how many arguments rhai will ask for +/// depends on what the *native* appends, which the wrapper cannot know. So such +/// a function is left to rhai, which has the body and can size the call itself. +pub(crate) fn takes_this(code: &[u8], chunk: Chunk, chains: &[Chain]) -> bool { + use crate::grain::bytecode::code::tag; + + let (entry, end) = (chunk.entry() as usize, chunk.end() as usize); + if end > code.len() || entry > end { + return false; + } + + crate::grain::bytecode::disassemble(&code[entry..end]).any(|(at, op)| { + match code[entry + at] { + tag::LOAD_THIS + | tag::LOAD_THIS_SHARED + | tag::REQUIRE_THIS + | tag::ASSIGN_THIS + | tag::ASSIGN_THIS_OP + | tag::CALL_THIS_REF + | tag::CALL_FN_PTR_ON_THIS => true, + // A chain says where it is rooted in the pool, not in the code. + tag::CHAIN => match op { + Op::Chain(index) => chains + .get(index as usize) + .map_or(false, |chain| matches!(chain.root, Root::This { .. })), + _ => false, + }, + _ => false, + } + }) +} + +/// Everything a program holds besides its code, gathered so the constructor +/// does not take ten positional arguments. +pub(crate) struct Parts<'a> { + pub positions: Positions, + pub residuals: Vec, + pub consts: Vec, + pub names: Strings<'a>, + pub tokens: Vec, + pub assign_ops: Vec, + pub chains: Vec, + pub switches: Vec, + pub lib: Option, + #[cfg(not(feature = "no_module"))] + pub resolver: Option>, + pub source: Option, +} + +impl<'a> Program<'a> { + pub(crate) fn new( + code: Code<'a>, + main: Chunk, + functions: Vec, + parts: Parts<'a>, + ) -> Self { + let makes_fn_pointers = makes_fn_pointers(&code); + let has_typed_methods = functions.iter().any(|f| f.this_type.is_some()); + + // Derived here so a program means the same whether it was compiled or + // loaded, and so the flag cannot disagree with the bytes it describes. + let mut functions = functions; + for function in &mut functions { + function.takes_this = takes_this(&code, function.chunk, &parts.chains); + } + + let mut program = Self { + code, + main, + functions, + max_stack: 0, + makes_fn_pointers, + has_typed_methods, + positions: parts.positions, + residuals: parts.residuals, + consts: parts.consts, + names: parts.names, + tokens: parts.tokens, + assign_ops: parts.assign_ops, + chains: parts.chains, + switches: parts.switches, + lib: parts.lib, + #[cfg(not(feature = "no_module"))] + resolver: parts.resolver, + source: parts.source, + }; + program.recompute_max_stack(); + program + } + + /// Copy the borrowed instructions, so this program outlives the artifact it + /// was read from. + /// + /// The opposite of the point, and only worth it when the buffer has to go. + #[must_use] + pub fn into_owned(self) -> Program<'static> { + Program { + code: Code::Owned(self.code.into_owned()), + main: self.main, + functions: self.functions, + max_stack: self.max_stack, + makes_fn_pointers: self.makes_fn_pointers, + has_typed_methods: self.has_typed_methods, + positions: self.positions, + residuals: self.residuals, + consts: self.consts, + names: self.names.into_owned(), + tokens: self.tokens, + assign_ops: self.assign_ops, + chains: self.chains, + switches: self.switches, + lib: self.lib, + #[cfg(not(feature = "no_module"))] + resolver: self.resolver, + source: self.source, + } + } + + /// Give up the artifact and share the program, so a native can be handed a + /// way back into it. + /// + /// What [`Vm::eval_with_callbacks`](crate::grain::Vm::eval_with_callbacks) takes. + /// Worth the copy only when [`makes_fn_pointers`](Self::makes_fn_pointers) + /// says a pointer can escape. + #[must_use] + pub fn into_shared(self) -> SharedProgram { + Shared::new(self.into_owned()) + } + + /// Check the chunk is internally consistent, returning the stack high water + /// it measured. + /// + /// Cheap enough to run on every compile, and the gate an artifact loaded + /// from a wire has to pass before the VM will touch it. + pub fn verify(&self) -> Result, crate::grain::bytecode::VerifyError> { + crate::grain::bytecode::verify(&self.code, &self.chunks(), self.pools()) + } + + /// Every chunk, main first, in the order they sit in the code. + fn chunks(&self) -> Vec { + core::iter::once(self.main) + .chain(self.functions.iter().map(|f| f.chunk)) + .collect() + } + + pub(crate) fn pools(&self) -> Pools<'_> { + Pools { + consts: self.consts.len(), + names: self.names.len(), + tokens: self.tokens.len(), + assign_ops: self.assign_ops.len(), + residuals: self.residuals.len(), + chains: &self.chains, + switches: &self.switches, + } + } + + /// Replace the compiler's upper-bound stack estimate with the verified high + /// water, so the VM reserves what the chunk uses rather than one slot per + /// instruction. + /// + /// A chunk that does not verify keeps its estimate: the VM is still safe + /// with a value that is too large, and [`Program::verify`] is where the + /// real failure should surface. + pub(crate) fn tighten_stack(&mut self) { + let Ok(high_water) = self.verify() else { + return; + }; + let mut measured = high_water.into_iter(); + if let Some(main) = measured.next() { + self.main.set_max_stack(main); + } + for (function, high_water) in self.functions.iter_mut().zip(measured) { + function.chunk.set_max_stack(high_water); + } + self.recompute_max_stack(); + } + + /// Every chunk's instructions, concatenated. + #[must_use] + pub fn code(&self) -> &[u8] { + &self.code + } + + /// The compiled script functions. + #[must_use] + pub fn functions(&self) -> &[Function] { + &self.functions + } + + /// The compiled function a call site resolves to, if there is one. + /// + /// Name and arity only, matching how rhai keys script functions. The name + /// is an index into the pool the call site also indexes, so equal names + /// have equal indices and this is two integer comparisons. + /// + /// Typed methods are invisible here. Rhai only ever tries a typed hash on a + /// *method* call (`func/call.rs:614`), so `fn .foo()` cannot be reached + /// as `foo()` — see [`Program::method`], which is the other door. + pub(crate) fn function(&self, name: u32, argc: usize) -> Option<&Function> { + self.functions + .iter() + .find(|f| f.name == name && f.params.len() == argc && f.this_type.is_none()) + } + + /// The compiled function a *method* call resolves to. + /// + /// `argc` excludes the receiver: `x.foo(1)` looks for the script function + /// `foo` of arity **one** and binds `this` to `x`, which is what the parser + /// hashes (`parser.rs:2128-2145`). That is the whole difference from + /// [`Program::function`], whose `argc` counts the receiver because the + /// rewrite it serves is function-call style. + /// + /// `typed` is the receiver's mapped type name. A function declared for it + /// wins, and an untyped one of the same name and arity is the fallback — + /// rhai's order, minus the hashing (`func/call.rs:614-629`). + pub(crate) fn method(&self, name: u32, argc: usize, typed: &str) -> Option<&Function> { + let matching = |f: &&Function| f.name == name && f.params.len() == argc; + + // Nearly every program has no typed method at all, and this is a linear + // scan on every method call — so the extra pass is bought only where + // there is something for it to find. + if self.has_typed_methods { + let found = self + .functions + .iter() + .find(|f| matching(f) && f.this_type.and_then(|t| self.name(t)) == Some(typed)); + if found.is_some() { + return found; + } + } + + self.functions + .iter() + .find(|f| matching(f) && f.this_type.is_none()) + } + + /// The compiled function a *pointer* resolves to. + /// + /// By name rather than by pool index, because a `FnPtr` carries a string — + /// it may have been built from one at run time. A linear scan, which at + /// these sizes beats a map and keeps the common indexed lookup untouched. + pub(crate) fn function_named(&self, name: &str, argc: usize) -> Option<&Function> { + self.functions.iter().find(|f| { + f.params.len() == argc && f.this_type.is_none() && self.name(f.name) == Some(name) + }) + } + + /// Whether this program can hand a function pointer to something that + /// might call it back. + /// + /// A compiled function lives in this program's own table and nowhere rhai + /// can see, so a native that calls a pointer — `map`, `filter` — cannot + /// reach one. Making it reachable means registering a wrapper, and rhai + /// requires a registered function to be `'static`, so the wrapper has to + /// own the program: [`Program::into_owned`] first, at the cost of the + /// borrowed-from-the-artifact loading that is the point of the format. + /// + /// This is how a host decides whether to pay that, without having to read + /// the script. False is the common answer and costs nothing. + #[must_use] + pub fn makes_fn_pointers(&self) -> bool { + self.makes_fn_pointers + } + + /// How much operand stack the deepest chunk needs. + /// + /// One reservation serves every frame, because a call pushes its operands + /// above the caller's rather than starting a stack of its own. Cached + /// rather than recomputed, since entering a frame reads it and entering a + /// frame is what a call does. + #[must_use] + pub fn max_stack(&self) -> u16 { + self.max_stack + } + + fn recompute_max_stack(&mut self) { + self.max_stack = self + .functions + .iter() + .map(|f| f.chunk.max_stack()) + .chain(core::iter::once(self.main.max_stack())) + .max() + .unwrap_or(0); + } + + pub(crate) fn constant(&self, index: u32) -> Option<&Dynamic> { + self.consts.get(index as usize) + } + + /// A name, borrowed from the artifact. Never allocates. + pub(crate) fn name(&self, index: u32) -> Option<&str> { + self.names.get(index) + } + + pub(crate) fn token(&self, index: u32) -> Option<&Token> { + self.tokens.get(index as usize) + } + + pub(crate) fn assign_op(&self, index: u32) -> Option<&AssignOp> { + self.assign_ops.get(index as usize) + } + + pub(crate) fn chain(&self, index: u32) -> Option<&Chain> { + self.chains.get(index as usize) + } + + pub(crate) fn chains(&self) -> &[Chain] { + &self.chains + } + + pub(crate) fn switch(&self, index: u32) -> Option<&Switch> { + self.switches.get(index as usize) + } + + /// The dispatch tables [`Op::Switch`](crate::grain::bytecode::Op::Switch) indexes. + /// + /// Public because a disassembly that leaves them out is misleading: a + /// switch's arms are reached only from its table, so without it they read + /// as unreachable code. + #[must_use] + pub fn switches(&self) -> &[Switch] { + &self.switches + } + + /// Where instruction `pc` came from, or `Position::NONE` if the table was + /// stripped or has nothing for it. + #[must_use] + pub fn position(&self, pc: usize) -> rhai::Position { + self.positions.get(pc) + } + + /// The whole position table, keyed on instruction address. + #[must_use] + pub fn positions(&self) -> &Positions { + &self.positions + } + + /// Drop the position table, returning it in its compact wire form. + /// + /// This is the separation the debug layer exists for: ship the program to + /// the device and keep what comes back here. Errors then arrive with no + /// position, and [`pos::resolve`](crate::grain::pos::resolve) turns the failing instruction + /// address back into one — see [`Program::attach_positions`] for the + /// inverse. + pub fn strip_positions(&mut self) -> Vec { + let table = self.positions.to_table(); + self.positions = Positions::Stripped; + table + } + + /// Put a stripped table back, so this program reports positions again. + /// + /// # Errors + /// + /// Refuses a malformed table, and refuses one whose addresses do not fit + /// this chunk — attaching another program's table would misreport every + /// error rather than reporting none, which is strictly worse. + pub fn attach_positions(&mut self, table: &[u8]) -> Result<(), TableError> { + self.positions = Positions::from_table(table, self.code.len())?; + Ok(()) + } + + pub(crate) fn consts(&self) -> &[Dynamic] { + &self.consts + } + + pub(crate) fn names(&self) -> &Strings<'a> { + &self.names + } + + pub(crate) fn tokens(&self) -> &[Token] { + &self.tokens + } + + pub(crate) fn assign_ops(&self) -> &[AssignOp] { + &self.assign_ops + } + + /// The top-level chunk, where execution starts. + #[must_use] + pub fn main(&self) -> &Chunk { + &self.main + } + + /// How many fragments rhai's walker still evaluates. + /// + /// Non-zero is the reason a program cannot yet be serialized. As a measure + /// of progress it is misleading on its own: lowering a statement often + /// splits one fragment into several smaller ones, so the count rises while + /// the work left shrinks. Use [`Program::residual_nodes`] for that. + #[must_use] + pub fn residual_count(&self) -> usize { + self.residuals.len() + } + + /// How many AST nodes are still inside fragments. + /// + /// This is the progress metric that only falls: it counts the tree that has + /// to survive into the artifact. + #[must_use] + pub fn residual_nodes(&self) -> usize { + let mut nodes = 0; + let path = &mut Vec::new(); + for residual in &self.residuals { + residual.walk(path, &mut |_| { + nodes += 1; + true + }); + } + nodes + } + + pub(crate) fn residual(&self, index: u32) -> Option<&Expr> { + self.residuals.get(index as usize) + } + + /// The construct that stopped this program being written, and where. + /// + /// A count of fragments is not something anyone can act on. This names the + /// first thing the compiler could not lower, so a validator can reject an + /// upload with the line to go and look at — which is what makes falling + /// back to shipping source a decision rather than a mystery. + #[must_use] + pub fn first_unsupported(&self) -> Option<(&'static str, rhai::Position)> { + let path = &mut Vec::new(); + let mut found: Option<(&'static str, rhai::Position)> = None; + + for residual in &self.residuals { + residual.walk(path, &mut |path| { + if found.is_some() { + return false; + } + if let Some(name) = path.last().and_then(unsupported_kind) { + found = Some((name, node_position(path.last().expect("just matched")))); + return false; + } + true + }); + if found.is_some() { + break; + } + } + + // A fragment made of nothing this recognises is still a fragment, so + // say so rather than reporting nothing wrong. + found.or_else(|| { + self.residuals + .first() + .map(|expr| ("an unlowered expression", expr.start_position())) + }) + } + + pub(crate) fn lib(&self) -> Option<&SharedModule> { + self.lib.as_ref() + } + + #[cfg(not(feature = "no_module"))] + pub(crate) fn resolver(&self) -> Option<&Shared> { + self.resolver.as_ref() + } + + pub(crate) fn source(&self) -> Option<&ImmutableString> { + self.source.as_ref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::grain::bytecode::{assemble, Op, Positions, Strings}; + + /// Names: 0 `f`, 1 `i64`, 2 `string`. + fn program_of(functions: &[(u32, Option, usize)]) -> Program<'static> { + // Every chunk is the same two instructions; only the table matters here. + let (code, _) = assemble(&[Op::Unit, Op::Return]).expect("must assemble"); + let whole = Chunk::new(0, code.len() as u32, 8); + + let functions = functions + .iter() + .map(|&(name, this_type, argc)| Function { + name, + params: vec![0; argc], + this_type, + takes_this: false, + chunk: whole, + }) + .collect(); + + Program::new( + code.into(), + whole, + functions, + Parts { + positions: Positions::default(), + residuals: Vec::new(), + consts: Vec::new(), + names: Strings::new(["f", "i64", "string"]), + tokens: Vec::new(), + assign_ops: Vec::new(), + chains: Vec::new(), + switches: Vec::new(), + lib: None, + #[cfg(not(feature = "no_module"))] + resolver: None, + source: None, + }, + ) + } + + /// Rhai tries the receiver's type first and falls back to the untyped + /// function of the same name and arity (`func/call.rs:614-629`). + #[test] + fn a_typed_method_wins_over_an_untyped_one_of_the_same_arity() { + let program = program_of(&[(0, Some(1), 0), (0, None, 0)]); + + assert_eq!(program.method(0, 0, "i64").unwrap().this_type, Some(1)); + // No function declared for a string, so the untyped one answers. + assert_eq!(program.method(0, 0, "string").unwrap().this_type, None); + } + + /// A typed method is only ever reached through a method call: rhai computes + /// the typed hash nowhere else, so `foo()` cannot find `fn .foo()`. + #[test] + fn a_typed_method_is_unreachable_in_call_style() { + let program = program_of(&[(0, Some(1), 0)]); + + assert!(program.function(0, 0).is_none()); + assert!(program.function_named("f", 0).is_none()); + assert!(program.method(0, 0, "i64").is_some()); + } + + #[test] + fn arity_is_matched_before_the_receiver_type() { + let program = program_of(&[(0, Some(1), 1), (0, None, 0)]); + + // The typed one takes an argument, so a no-argument call is the untyped. + assert_eq!(program.method(0, 0, "i64").unwrap().this_type, None); + assert_eq!(program.method(0, 1, "i64").unwrap().this_type, Some(1)); + } + + #[test] + fn a_receiver_type_survives_the_round_trip() { + let program = program_of(&[(0, Some(1), 0), (0, None, 0)]); + + let bytes = program.write().expect("must be writable"); + let reloaded = Program::read(&bytes).expect("must load"); + + let typed: Vec<_> = reloaded.functions().iter().map(|f| f.this_type).collect(); + assert_eq!(typed, vec![Some(1), None]); + assert_eq!(reloaded.method(0, 0, "i64").unwrap().this_type, Some(1)); + } +} diff --git a/src/grain/vm/callback.rs b/src/grain/vm/callback.rs new file mode 100644 index 000000000..4eea529f2 --- /dev/null +++ b/src/grain/vm/callback.rs @@ -0,0 +1,143 @@ +//! Reaching a compiled chunk from inside a native function. +//! +//! `[1, 2, 3].map(|x| x * 2)` leaves this VM in the middle of a call. `map` is +//! rhai's, and the pointer it calls back is resolved by rhai's dispatch, which +//! looks in `global.lib` and the engine's modules. Our chunks are in neither: +//! [`Op::Call`](crate::bytecode::Op::Call) finds one by a name *index* that +//! only the compiler and the call site share, and a `FnPtr` carries a string. +//! +//! So a program that hands a pointer out registers one native wrapper per +//! compiled function for the length of the run. Direct dispatch is untouched — +//! this is somewhere for rhai to look, not somewhere we look. +//! +//! # What being a native costs +//! +//! Rhai reaches its own closures through a `Fn*` pointer carrying the body, and +//! that shortcut is what these wrappers cannot have. Two consequences, both +//! measured rather than assumed: +//! +//! * **Argument order, for a capturing closure called by a native that binds +//! `this`.** A capture is a curried value, and `_call_with_extra_args` +//! (`types/fn_ptr.rs:573`) tries `[this] ++ curry ++ args` first for anything +//! that is not a `Fn*` pointer. That shape is never right, and it is what a +//! wrapper answers to. Stock rhai does the same thing to its own name-only +//! pointers — `stock_rhai_does_the_same_to_its_own_native_pointers` in +//! `tests/callback.rs` reproduces it with none of this involved — so the fix +//! is not here; it is upstream, or in not currying captures at all. +//! * **Speed, and call budget.** Rhai resolves a wrapper by name and type on +//! every element, from a cache it builds fresh per crossing, where its own +//! pointer skips resolution entirely. `native callbacks` in +//! `examples/bench.rs` measures 0.34x — the one case the VM loses — and the +//! two extra dispatch layers cost 5 call levels per crossing against the +//! walker's 2. +//! +//! Neither touches a pointer called directly from compiled code, which is +//! `Op::CallFnPtr` and never comes through here. + +use core::any::TypeId; +use core::mem; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +use crate::{ + func::RhaiFunc, Dynamic, FuncRegistration, ImmutableString, Module, NativeCallContext, Shared, +}; + +use super::{malformed, Vm, VmResult}; +use crate::grain::program::SharedProgram; + +/// The most parameters a wrapper is registered for. +/// +/// A wrapper takes `Dynamic` throughout, and rhai only reaches a `Dynamic` +/// parameter by permuting the call's own argument types towards it — a search +/// it caps at `MAX_DYNAMIC_PARAMETERS`, 16 (`func/call.rs:235`). A wider +/// wrapper would be registered and never found, so it is left out rather than +/// silently dead. Direct dispatch has no such bound; this limits only what a +/// native can call back into. +const MAX_PARAMS: usize = 16; + +/// A wrapper per compiled function, for rhai to resolve a pointer against. +/// +/// Built once per run rather than cached on the program: the closures hold the +/// program, so anything the program held back would be a cycle. +pub(super) fn wrappers(program: &SharedProgram) -> Module { + let mut module = Module::new(); + + // What rhai reports as the source of a function it found here. Its own + // script library is the AST's, so this is the same string by the same + // route. + if let Some(source) = program.source() { + module.set_id(source.clone()); + } + + for function in program.functions() { + let arity = function.params.len(); + if arity > MAX_PARAMS { + continue; + } + // A `this`-taking chunk cannot be reached this way. A wrapper is + // registered at one arity, and how many arguments rhai asks for depends + // on what the *native* appends beside the receiver — `map` adds an + // index, `reduce` adds the running result — which the wrapper has no way + // to know. Rhai's own pointer carries the body and sizes the call from + // its declared arity (`types/fn_ptr.rs:501-535`); a name-only pointer, + // which is all a wrapper can be, cannot. So these are left to rhai, + // and `Program::needs_walker` is what keeps its copy alive for them. + if function.takes_this { + continue; + } + let Some(name) = program.name(function.name) else { + continue; + }; + + let owner = program.clone(); + let called: ImmutableString = name.into(); + + // One closure for every arity, rather than the fixed-arity shapes + // `Module::set_native_fn` generates. `Dynamic` parameters throughout + // mean the types never have to line up, only the count. + let wrapper = move |context: Option, args: &mut [&mut Dynamic]| { + invoke(&owner, &called, context.as_ref(), args) + }; + + FuncRegistration::new(name) + .in_internal_namespace() + .set_into_module_raw( + &mut module, + vec![TypeId::of::(); arity], + RhaiFunc::Pure { + func: Shared::new(wrapper), + has_context: true, + is_pure: true, + is_volatile: false, + }, + ); + } + + module +} + +/// Run one chunk for a native that called back into us. +fn invoke( + program: &SharedProgram, + name: &str, + context: Option<&NativeCallContext>, + args: &mut [&mut Dynamic], +) -> VmResult { + // Registered with `has_context`, so rhai always supplies one. + let context = + context.ok_or_else(|| malformed("a callback wrapper was given no context".into()))?; + + // Taken rather than cloned, as every registered function does: the + // arguments are the caller's to give away, and it has already copied + // anything it still needs. + let values: Vec = args.iter_mut().map(|arg| mem::take(*arg)).collect(); + + Vm::reentrant(context).call_function( + program, + name, + values, + context.call_level(), + context.call_position(), + ) +} diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs new file mode 100644 index 000000000..67c402667 --- /dev/null +++ b/src/grain/vm/mod.rs @@ -0,0 +1,3953 @@ +use core::mem; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + +use crate::engine::{FN_IDX_GET, FN_IDX_SET}; +#[cfg(not(feature = "unchecked"))] +use crate::eval::calc_data_sizes; +use crate::func::{get_builtin_binary_op_fn, get_builtin_op_assignment_fn}; +use crate::packages::string_basic::print_with_func; +use crate::types::dynamic::DynamicWriteLock; +use crate::types::fn_ptr::FnPtrType; +// `Variant` is only re-exported from the crate root under `internals`, so it +// comes from where it is defined. +use crate::ast::Expr; +use crate::types::dynamic::Variant; +use crate::{ + eval::Caches, eval::GlobalRuntimeState, CallFnOptions, Dynamic, Engine, EvalAltResult, + EvalContext, Scope, +}; +use crate::{ + Array, FnPtr, ImmutableString, Map, NativeCallContext, Position, ThinVec, FUNC_TO_STRING, INT, +}; + +mod callback; + +use crate::grain::bytecode::{code, AssignOp, Chain, Receiver, Root, Step, Tail}; +use crate::grain::program::{Program, SharedModule, SharedProgram}; + +/// rhai's own `RhaiResult`, which it does not re-export. +pub type VmResult = Result>; + +/// Whether a value is a shared cell. +/// +/// Sharing is how closures capture, so under `no_closure` there are no cells, +/// `Dynamic` has no `is_shared` to call, and the answer is a constant. The +/// opcodes that create and read cells are compiled out with it. +#[cfg(not(feature = "no_closure"))] +macro_rules! is_shared { + ($value:expr) => { + $value.is_shared() + }; +} +#[cfg(feature = "no_closure")] +macro_rules! is_shared { + ($value:expr) => {{ + let _ = &$value; + false + }}; +} + +/// A chunk that does not agree with itself — a slot past the end of the scope, +/// an index into a pool that has no such entry, an operand stack that ran dry. +/// +/// Reachable only through a compiler bug or a corrupt artifact, never through +/// anything a script can express. Verification turns most of these into load +/// time failures; the rest surface as runtime errors rather than panics, so a +/// bad chunk cannot take the host down. +/// Build a [`NativeCallContext`] from its parts. +/// +/// `NativeCallContext::new_with_all_fields` is the obvious spelling but is +/// `#[cfg(not(feature = "no_module"))]`. The `From` impl over the same five +/// fields is not gated and assigns exactly the same ones, so this works in +/// either configuration without a cfg of its own. +fn native_context<'a>( + engine: &'a Engine, + fn_name: &'a str, + source: Option<&'a str>, + global: &'a GlobalRuntimeState, + pos: Position, +) -> NativeCallContext<'a> { + NativeCallContext::from((engine, fn_name, source, global, pos)) +} + +/// Stamp the call site on an error that passes through a function boundary +/// unwrapped, as rhai does for exits and system exceptions +/// (`func/script.rs:134`). +fn reposition(mut err: Box, pos: Position) -> Box { + err.set_position(pos); + err +} + +/// Stamp the site on an error that arrived without one. +/// +/// Dispatch raises `ErrorFunctionNotFound` at `Position::NONE` and leaves +/// positioning to the caller, which has the expression that failed. Unlike +/// [`reposition`] this never overwrites a position the callee already set. +fn positioned(err: Box, pos: Position) -> Box { + if err.position().is_none() { + reposition(err, pos) + } else { + err + } +} + +/// Stamp the call site on anything a dispatched call came back with. +/// +/// This is `fill_position`, which rhai applies to the whole of +/// `exec_native_fn_call` — the callee not being found, the call being refused, +/// and the error a native *returned* alike (`func/call.rs:365`, `:406`, +/// `:413`). `call_fn_raw` has no position to give, so all of them arrive bare. +/// +/// What looks like a counter-example is not one: `1 / 0` reports +/// `ErrorArithmetic` with no position at all, because under `fast_operators` a +/// binary operator returns the built-in's error without going through here +/// (`func/call.rs:1798`). The VM's own fast path skips it for the same reason. +fn dispatch_failure(err: Box, pos: Position) -> Box { + positioned(err, pos) +} + +/// A scope entry, addressed the way whatever wants it was written. +/// +/// A slot always names one and a name may name nothing, which is the whole of +/// the difference between the two at run time — for a [`Receiver`] and for a +/// [`Root`] alike. +#[derive(Clone, Copy)] +enum Site<'a> { + Slot(usize), + Name(&'a str), +} + +/// What a chain turned out to be rooted at. +/// +/// Rhai draws this line in `search_namespace`, which hands back a `Target`: a +/// scope entry becomes a reference to write through, and a resolver's answer or +/// a module's constant becomes a read-only temporary (`eval/expr.rs:120-155`). +/// Which one a [`Root::Named`] is cannot be known until it is looked up. +enum RootAt<'a> { + /// A scope entry. The only root a chain writes back into, and only when it + /// is not a constant. + Place(Site<'a>), + /// A value with a name but no entry behind it — a resolver's answer, or a + /// module's constant. + Constant, + /// The frame's receiver, which is a register rather than a scope entry but + /// writes back for the same reason a [`RootAt::Place`] does. + This, + /// A value with no name either: `[1, 2].len()`, `f().x`. Nothing can be + /// assigned to one, because rhai's parser refuses it outright. + Temporary, +} + +/// A chain's root, looked up. +struct ChainRoot<'a> { + at: RootAt<'a>, + /// The value to walk. + /// + /// **Read-only if rhai's `Target` would have been**, which is not + /// decoration: cloning a `Dynamic` marks the copy read-write however the + /// original was (`types/dynamic.rs:822`), and the access mode is the only + /// thing standing between a `const` and a method that mutates it — + /// `exec_native_fn_call` refuses a non-pure function whose first argument + /// is read-only (`func/call.rs:405`). + value: Dynamic, + /// Where to blame a refusal: the variable for a name, the chain otherwise. + pos: Position, +} + +/// What one indexing step managed. +enum Indexed { + /// Taken through a reference: the value, and whether anything wrote. + Done(Dynamic, bool), + /// There was no reference to take. Carries the value back out, because the + /// caller cannot touch the container until this borrow has ended. + NoReference(Dynamic), +} + +/// What a chain's root is called, for the two errors that name it. +/// +/// `None` for a temporary, which has no name to give — and neither error can +/// reach one: nothing assigns through a temporary, and flattening it on the way +/// in means there is no cell left to contend for. +/// +/// `this` *can* reach both and has no name either, so it answers with the empty +/// one rather than with nothing. That is rhai's own answer: `Expr::ThisPtr` +/// carries no name, so assigning through a read-only receiver is +/// `ErrorAssignmentToConstant("")` (`eval/stmt.rs:118-122`). Answering `None` +/// here would report a malformed chunk instead. +fn root_name<'p>(program: &'p Program, chain: &Chain) -> Option<&'p str> { + match chain.root { + Root::Local { name, .. } | Root::Named { name, .. } => program.name(name), + Root::This { .. } => Some(""), + Root::Temporary => None, + } +} + +/// The op-assignment a chain ends with, resolved out of the pool. +fn chain_op<'p>( + program: &'p Program, + chain: &Chain, +) -> Result, Box> { + let Tail::Assign { op: Some(op) } = &chain.tail else { + return Ok(None); + }; + program + .assign_op(*op) + .map(Some) + .ok_or_else(|| malformed(format!("no op-assignment {op}"))) +} + +/// One `for` loop in progress. +/// +/// The count is here rather than in a local because rhai keeps it outside the +/// scope too, and checks it for overflow before writing it — a loop long +/// enough to wrap the counter is an error rather than a wrap +/// (`eval/stmt.rs:729`). +struct Iteration { + items: Box>, + /// The index of the item last handed out, starting one below the first. + count: INT, +} + +/// A scope entry as a place to write, seeing through a shared cell. +/// +/// Rhai reaches a variable through a `Target`, whose shared arm hands over the +/// cell's guard rather than the cell (`eval/target.rs:409-422`), so an +/// assignment lands where every closure holding that cell can see it. Writing +/// the slot itself would replace the cell and quietly sever them — the value +/// would be right and the aliasing dead. +/// +/// For an ordinary value `write_lock` is a downcast to itself, so the common +/// case pays nothing. Rhai's own for-loop does this with `.unwrap()` and +/// panics on a contended cell; a VM that promises errors instead of panics +/// reports `ErrorDataRace`, as `Target` does. +fn place<'a>( + entry: &'a mut Dynamic, + name: &str, + pos: Position, +) -> Result, Box> { + entry + .write_lock::() + .ok_or_else(|| Box::new(EvalAltResult::ErrorDataRace(name.to_string(), pos))) +} + +/// Turn the two control-flow errors back into the value they carry. +/// +/// Rhai unwinds `return` and `exit` as errors rather than returning them; +/// `eval_global_statements` is where they turn back into values, and anything +/// entering a program from outside has to do the same. +fn unwind_exit(result: VmResult) -> VmResult { + result.or_else(|err| match *err { + EvalAltResult::Return(out, ..) | EvalAltResult::Exit(out, ..) => Ok(out), + _ => Err(err), + }) +} + +fn missing(name: &str, pos: Position) -> Box { + Box::new(EvalAltResult::ErrorVariableNotFound(name.to_string(), pos)) +} + +fn malformed(detail: String) -> Box { + Box::new(EvalAltResult::ErrorRuntime( + format!("malformed chunk: {detail}").into(), + Position::NONE, + )) +} + +/// Executes a [`Program`] against an `Engine`. +/// +/// Holds one `GlobalRuntimeState` and one `Caches` for its whole lifetime, so +/// the function-resolution cache survives across calls. That matters: the +/// reentrant helpers rhai exposes to native functions build a fresh +/// `Caches::new()` per call (`func/native.rs:519`), which would throw away +/// resolution work on every dispatch. +pub struct Vm<'e> { + engine: &'e Engine, + global: GlobalRuntimeState, + caches: Caches, + stack: Vec, + /// One entry per `for` loop currently running. + /// + /// Not on the operand stack, because an iterator is not a `Dynamic`. A + /// frame truncates this to what it found on entry, so a `return` or an + /// escaping error drops whatever its loops were holding without the + /// compiler emitting anything. + iterators: Vec, + /// One entry per `try` region currently armed or catching. Frame-floored + /// the same way the iterators are, so an error in a called function can + /// never find its caller's handler and jump into another chunk. + handlers: Vec, + /// The running data-size total of each literal currently being built, + /// innermost last. + /// + /// One entry per array or map literal under construction, so + /// `[a, [b, c], d]` keeps the inner total separate from the outer. + /// Truncated per frame, as the iterators are, so an error part way through + /// a literal leaves nothing behind. + sizes: Vec<(usize, usize, usize)>, + /// Where the scope goes back to if an error escapes the running frame. + /// + /// Set by [`Op::Checkpoint`](crate::bytecode::Op::Checkpoint) at each + /// top-level statement of a chunk, saved and restored per frame. See + /// [`Vm::execute`]. + unwind_floor: usize, + /// The receiver bound to the frame currently running. + /// + /// Owned rather than borrowed: the value a binder has to hand over always + /// lives in `stack` or in a caller's `Scope`, and neither can lend a `&mut` + /// across the `&mut self` call that runs the callee. So the binder moves it + /// in, the frame owns it, and [`Vm::call_compiled_with_this`] hands it back + /// for the binder to put where it came from. + /// + /// Saved and restored per *call* rather than per frame, which is what makes + /// `this` never inherited: a plain nested call passes `None` and gets the + /// caller's back on the way out, reproducing `func/call.rs:669` without a + /// conditional anywhere. + this: Option, + fault_pc: Option, +} + +/// Take a receiver for a callee to own, and say whether it goes back. +/// +/// The rule is `chain_root`'s `walkable` (see [`Vm::chain_root`]) plus the +/// question of ownership. A read-only receiver is *cloned*, because `take` +/// would leave `UNIT` behind and strip the constness the caller's slot is +/// carrying; nothing is written back, since rhai refuses the mutation rather +/// than making it. Anything else is taken, as `call_compiled_body` already +/// takes a call's arguments — a receiver is not shared with anything the callee +/// can reach, and taking it saves a deep clone of an array or a map per call. +/// +/// A shared receiver is taken like any other: the taken value *is* the cell, so +/// `is_shared(this)` stays true inside the body and a write lands where every +/// other holder can see it. It still goes back, because taking it left `UNIT` +/// where the cell was. +fn bind_this(receiver: &mut Dynamic) -> (Dynamic, bool) { + if receiver.is_read_only() { + (receiver.clone().into_read_only(), false) + } else { + (mem::take(receiver), true) + } +} + +/// Put a receiver back where [`bind_this`] took it from. +/// +/// Unconditional on how the call ended. Rhai reaches `this` through a pointer +/// into the caller's storage, so a body that mutates and then raises has +/// already written; a binder that only restored on success would discard +/// exactly that. +fn unbind_this(receiver: &mut Dynamic, taken: Option, write_back: bool) { + if let (true, Some(value)) = (write_back, taken) { + *receiver = value; + } +} + +/// A `try` region. +struct Handler { + target: usize, + catch_var: Option, + /// Where the three stacks were when the region was entered. An error can + /// be raised at any depth of all three, and the catch block has to begin + /// where the `try` did. + operands: usize, + scope_len: usize, + iters: usize, + /// Set once the catch block is running, holding the error it caught. That + /// is what a bare `throw;` in the catch block re-raises, and its presence + /// is what tells an escaping error it is leaving a catch rather than + /// entering one. + caught: Option>, +} + +impl<'e> Vm<'e> { + /// A VM that dispatches through `engine`. + #[must_use] + pub fn new(engine: &'e Engine) -> Self { + Self { + engine, + global: engine.new_global_runtime_state(), + caches: Caches::new(), + stack: Vec::new(), + iterators: Vec::new(), + handlers: Vec::new(), + sizes: Vec::new(), + unwind_floor: 0, + this: None, + fault_pc: None, + } + } + + /// A `Vm` for a call arriving from inside a native function. + /// + /// Reproduces what rhai does at every reentrant boundary + /// (`func/native.rs:516-519`, `types/fn_ptr.rs:451-454`): the caller's + /// runtime state is *cloned* rather than shared, and the resolution cache + /// starts empty. The clone is what carries the imported modules, the source + /// name and — the part that matters here — the function library holding the + /// wrappers, so a closure reached from a native can hand out a pointer of + /// its own. + /// + /// The empty `Caches` is the cost, and it is the one thing a `Vm` normally + /// exists to avoid. It cannot be helped: the outer `Vm` is borrowed by the + /// frame still running beneath this one. Rhai pays the same on its own + /// callbacks — but it also skips resolution entirely for a pointer that + /// carries its body, which is why a crossing measures 0.34x. See the + /// `callback` module. + /// + /// Operation counting has the same shape and the same reason: increments + /// inside the callback land on the clone and are lost when it drops, as + /// they are for any reentrant call rhai makes. + #[must_use] + pub fn reentrant(context: &'e NativeCallContext<'_>) -> Self { + Self { + engine: context.engine(), + global: context.global_runtime_state().clone(), + caches: Caches::new(), + stack: Vec::new(), + iterators: Vec::new(), + handlers: Vec::new(), + sizes: Vec::new(), + unwind_floor: 0, + // A crossing carries no receiver: rhai binds one only where it + // dispatches a method, and this arrives through `call_fn_raw`. + this: None, + fault_pc: None, + } + } + + /// Which instruction the last run failed at, if it failed. + /// + /// This is what a stripped program reports instead of a position. The host + /// that kept the table resolves it with `rhaigrain_pos::resolve`, so a + /// device can stay silent about where its source was and still produce a + /// diagnostic someone can act on. + /// + /// Cleared at the start of every run, so it always describes the most + /// recent one. + #[must_use] + pub fn fault_pc(&self) -> Option { + self.fault_pc + } + + /// Run a program, returning its value. + /// + /// Mirrors `Engine::eval_ast_with_scope_raw`: the program's function + /// library, module resolver and source name are installed for the duration + /// and restored afterwards, so a `Vm` reused across programs does not leak + /// one program's definitions into the next. + /// Call one compiled function by name, with arguments already evaluated. + /// + /// The entry point a native needs. `Op::Call` reaches a chunk through the + /// name *index* it shares with the call site, which a caller from outside + /// does not have — a `FnPtr` carries a string, and so does rhai when it + /// dispatches. This is the same call by the other key. + /// + /// `level` is the caller's call depth, so `max_call_levels` still counts + /// across a boundary that leaves this VM and comes back. Left unthreaded, + /// a closure calling itself through `map` would recurse until the stack + /// went rather than until the limit did. + /// + /// # Errors + /// + /// `ErrorFunctionNotFound` if no compiled function has that name and + /// arity, and whatever the function itself raises otherwise. + pub fn call_function( + &mut self, + program: &Program, + name: &str, + args: Vec, + level: usize, + pos: Position, + ) -> VmResult { + self.call_function_with_this(program, name, args, level, pos, None) + .0 + } + + /// The same, against a receiver the callee owns for the duration. + /// + /// The receiver comes back however the call ended, for the caller to put + /// where it took it from — see [`bind_this`] and [`unbind_this`]. + fn call_function_with_this( + &mut self, + program: &Program, + name: &str, + args: Vec, + level: usize, + pos: Position, + this: Option, + ) -> (VmResult, Option) { + let Some(function) = program.function_named(name, args.len()) else { + return ( + Err(Box::new(EvalAltResult::ErrorFunctionNotFound( + format!("{name} ({} args)", args.len()), + pos, + ))), + this, + ); + }; + let (params, chunk) = (function.params.clone(), function.chunk); + + // `call_compiled` takes its arguments off the operand stack, where a + // compiled call site would already have put them. + let first = self.stack.len(); + self.stack.extend(args); + + let restore = mem::replace(&mut self.global.level, level); + let (result, this) = + self.call_compiled_with_this(program, name, ¶ms, chunk, first, pos, this); + self.global.level = restore; + + self.stack.truncate(first); + (result, this) + } + + /// Call one compiled function by name, instead of running the whole + /// program. + /// + /// Mirrors [`Engine::call_fn`](crate::Engine::call_fn), including that the + /// program's body runs first — a function usually needs what the top level + /// declared. [`call_fn_with_options`](Self::call_fn_with_options) turns + /// that off. + /// + /// # Errors + /// + /// `ErrorFunctionNotFound` if no compiled function has that name and + /// arity, `ErrorMismatchOutputType` if the result is not a `T`, and + /// whatever the function itself raises. + pub fn call_fn( + &mut self, + scope: &mut Scope, + program: &Program, + name: impl AsRef, + args: impl crate::FuncArgs, + ) -> Result> { + self.call_fn_with_options(CallFnOptions::new(), scope, program, name, args) + } + + /// The same, with rhai's [`CallFnOptions`](crate::CallFnOptions). + /// + /// Three of the five options mean something here: + /// + /// * `eval_ast` runs the program's main chunk before the call, so what the + /// top level declares is in scope for it. On by default, as in rhai. + /// * `rewind_scope` truncates the scope back afterwards. On by default. + /// * `tag` sets the evaluation's custom state. + /// + /// `this_ptr` binds the callee's receiver, as it does in rhai. A write + /// through `this` lands in the pointer's own `Dynamic` — including when the + /// call goes on to fail, because rhai reaches `this` through the caller's + /// storage and a body that mutates and then raises has already written. + /// + /// `in_all_namespaces` is ignored: this looks only in the program's own + /// compiled functions. + /// + /// # Errors + /// + /// As [`call_fn`](Self::call_fn). + pub fn call_fn_with_options( + &mut self, + options: CallFnOptions, + scope: &mut Scope, + program: &Program, + name: impl AsRef, + args: impl crate::FuncArgs, + ) -> Result> { + let name = name.as_ref(); + + let mut arg_values = Vec::new(); + args.parse(&mut arg_values); + + let orig_scope_len = scope.len(); + let mut this_ptr = options.this_ptr; + if let Some(tag) = options.tag { + self.global.tag = tag; + } + + // The pointer is the host's and outlives the call, so unlike every + // other binder this one can hold it across the whole thing. + let bound = this_ptr.as_deref_mut().map(bind_this); + let (this, write_back) = bound.map_or((None, false), |(v, w)| (Some(v), w)); + + // The program's environment stays installed for the *call*, not only for + // the main chunk that may precede it: the function being called can + // reach whatever the compiler left rhai to interpret, and rhai looks for + // it in `global.lib`. + let mut evaluated = Ok(()); + let (result, returned) = self.with_environment(program, None, |vm| { + if options.eval_ast { + // Run for the scope it leaves behind; the body's own value is + // not what the caller asked for. The main chunk gets no + // receiver, as `eval_global_statements` does not either. + evaluated = unwind_exit(vm.run_main(program, scope)).map(|_| ()); + if evaluated.is_err() { + return (Ok(Dynamic::UNIT), this); + } + } + vm.call_function_with_this(program, name, arg_values, 0, Position::NONE, this) + }); + + // Before the errors below, both of them: rhai's mutation through `this` + // survives a failed call, and survives one whose result is the wrong + // type just as much. + if let Some(slot) = this_ptr { + unbind_this(slot, returned, write_back); + } + evaluated?; + + if options.rewind_scope { + scope.rewind(orig_scope_len); + } + + result?.try_cast_result().map_err(|value| { + Box::new(EvalAltResult::ErrorMismatchOutputType( + self.engine + .map_type_name(core::any::type_name::()) + .into(), + self.engine.map_type_name(value.type_name()).into(), + Position::NONE, + )) + }) + } + + /// Evaluate a program's main chunk against `scope`, yielding its value. + /// + /// The scope is the caller's, as it is for + /// [`Engine::eval_ast_with_scope`](crate::Engine::eval_ast_with_scope): + /// what the program declares at the top level is left in it, and what the + /// caller put there beforehand is visible to the program. + /// + /// # Errors + /// + /// Whatever the program raises, and `ErrorRuntime` for a malformed one. + pub fn eval_with_scope(&mut self, scope: &mut Scope, program: &Program) -> VmResult { + self.run_with(program, scope, None) + } + + /// The same against a scope of its own, for a program that needs none. + /// + /// # Errors + /// + /// As [`eval_with_scope`](Self::eval_with_scope). + pub fn eval(&mut self, program: &Program) -> VmResult { + self.eval_with_scope(&mut Scope::new(), program) + } + + /// Evaluate a program against `scope` for its effects, discarding its value. + /// + /// # Errors + /// + /// As [`eval_with_scope`](Self::eval_with_scope). + pub fn run_with_scope( + &mut self, + scope: &mut Scope, + program: &Program, + ) -> Result<(), Box> { + self.eval_with_scope(scope, program).map(|_| ()) + } + + /// The same against a scope of its own. + /// + /// # Errors + /// + /// As [`eval_with_scope`](Self::eval_with_scope). + pub fn run(&mut self, program: &Program) -> Result<(), Box> { + self.run_with_scope(&mut Scope::new(), program) + } + + /// Evaluate a program that hands function pointers to native functions. + /// + /// The same run, plus one native wrapper per compiled function registered + /// for its duration, so a pointer this program creates resolves when rhai + /// dispatches it — `let a = [1, 2]; a.map(|x| x * 2)` is `map` calling us + /// back, and `map` looks the pointer up its own way. See the `callback` + /// module. + /// + /// Only worth the owned program when [`Program::makes_fn_pointers`] says a + /// pointer can escape; [`eval_with_scope`](Self::eval_with_scope) is + /// otherwise identical and copies nothing. A program that needs this and + /// does not get it still runs — the pointer simply fails to resolve, as + /// `ErrorFunctionNotFound`, at the point the native tries to call it. + /// + /// Read the `callback` module before relying on it: a crossing is slower than the + /// walker, and a *capturing* closure handed to a native that binds `this` + /// arrives with its arguments rotated. + /// + /// Named `eval_` rather than `run_` because it yields the program's value; + /// rhai has no `Engine` method to mirror here, so the crate's own rule is + /// the one that applies. + /// + /// # Errors + /// + /// As [`eval_with_scope`](Self::eval_with_scope). + pub fn eval_with_callbacks(&mut self, scope: &mut Scope, program: &SharedProgram) -> VmResult { + let wrappers = + (!program.functions().is_empty()).then(|| callback::wrappers(program).into()); + self.run_with(program, scope, wrappers) + } + + fn run_with( + &mut self, + program: &Program, + scope: &mut Scope, + wrappers: Option, + ) -> VmResult { + let result = self.with_environment(program, wrappers, |vm| vm.run_main(program, scope)); + unwind_exit(result) + } + + /// What a program contributes to `global`, in place for the duration of `f`. + /// + /// Everything entering a program from outside needs this, not only its main + /// chunk: a function reached through [`call_fn`](Self::call_fn) can call + /// whatever the compiler left rhai to interpret, and rhai looks for it in + /// `global.lib`. + fn with_environment( + &mut self, + program: &Program, + wrappers: Option, + f: impl FnOnce(&mut Self) -> T, + ) -> T { + let orig_source = mem::replace(&mut self.global.source, program.source().cloned()); + let orig_lib_len = self.global.lib.len(); + if let Some(lib) = program.lib() { + self.global.lib.push(lib.clone()); + } + // Last, so the search — which runs in reverse — reaches a compiled + // function before whatever the compiler left rhai to interpret. + if let Some(wrappers) = wrappers { + self.global.lib.push(wrappers); + } + #[cfg(not(feature = "no_module"))] + let orig_resolver = mem::replace( + &mut self.global.embedded_module_resolver, + program.resolver().cloned(), + ); + + let result = f(self); + + #[cfg(not(feature = "no_module"))] + { + self.global.embedded_module_resolver = orig_resolver; + } + self.global.lib.truncate(orig_lib_len); + self.global.source = orig_source; + + result + } + + /// The main chunk, against an environment the caller has already installed. + fn run_main(&mut self, program: &Program, scope: &mut Scope) -> VmResult { + self.fault_pc = None; + let mut pc = program.main().entry() as usize; + // Slots are indices into the caller's scope, so a caller that arrives + // with variables already in it shifts every one of them. + let base = scope.len(); + let result = self.execute(program, scope, *program.main(), base, &mut pc); + if result.is_err() { + self.fault_pc = Some(pc); + } + result + } + + fn pop(&mut self) -> Result> { + self.stack + .pop() + .ok_or_else(|| malformed("operand stack underflow".to_string())) + } + + /// `reached` tracks the instruction being executed, so a failure can be + /// attributed to one. It is what a stripped program reports in place of a + /// position. + /// Call a chunk this compiler produced, reproducing `call_script_fn` + /// (`func/script.rs:24`) step for step. + /// + /// The parts that are not obvious, and that the differential corpus is + /// what proves: arguments are *taken* out of the caller's stack slots + /// rather than cloned, the depth check happens after the level is + /// incremented, and only errors that are neither a `return` nor a system + /// exception get wrapped in `ErrorInFunctionCall`. + /// Walk `a.b[i].c`, reading it or assigning to it. + /// + /// The reason this is one instruction and a recursion rather than a + /// sequence: every level holds a `&mut` into the level above, exactly as + /// rhai does (`eval/chaining.rs:659`). For a map or an array that borrow is + /// the whole story — the mutation lands in the container and no write-back + /// is needed. Doing it on the operand stack instead would mutate a copy. + /// + /// Write-back is only for the levels where a borrow was not possible: + /// a getter on a host type hands back a value, and rhai calls the setter + /// afterwards if the sub-chain was a method call. `changed` reproduces + /// that, and it is deliberately coarse in the same way rhai's is — rhai's + /// flag is `func.is_method()`, "does the resolved function take its + /// receiver by reference", not "did it actually write". + #[inline(never)] + fn run_chain( + &mut self, + program: &Program, + chain: &Chain, + scope: &mut Scope, + base: usize, + pos: Position, + ) -> VmResult { + // Step operands were pushed first, then the root if it is one that has + // to be evaluated, then the value being assigned. + let operands_at = self + .stack + .len() + .checked_sub(chain.consumes()) + .ok_or_else(|| malformed("chain with too few operands".to_string()))?; + + let ChainRoot { + at, + value: mut root, + pos: root_pos, + } = self.chain_root(program, chain, scope, base, operands_at, pos)?; + + // Read-only is what refuses an assignment, not the absence of a place: + // a module's constant is neither, so rhai assigns into the copy and + // discards it. Only a `const` and a resolver's answer are refused, and + // both are read-only for that reason. + // + // A temporary is separate again — rhai's parser refuses `f().x = 1` + // outright, so one reaches here only from a chunk this compiler did + // not build. + let value = match (chain.assigns(), &at) { + (false, _) => None, + (true, RootAt::Temporary) => { + return Err(malformed("chain assigns through a temporary root".into())) + } + (true, _) if root.is_read_only() => { + let name = root_name(program, chain) + .ok_or_else(|| malformed("no chain root name".to_string()))?; + return Err(Box::new(EvalAltResult::ErrorAssignmentToConstant( + name.to_string(), + root_pos, + ))); + } + // Rhai flattens the right-hand side before assigning, so a shared + // cell is copied out rather than aliased in. + (true, _) => Some(self.stack[self.stack.len() - 1].clone().flatten()), + }; + + let mut operands: Vec = + self.stack[operands_at..operands_at + chain.operands as usize].to_vec(); + + // A shared cell cannot be walked directly. `get_indexed_mut` refuses + // one outright — `unreachable!("cannot handle shared values")`, + // `eval/chaining.rs:461` — because rhai always reaches a root through + // a `Target`, whose shared arm hands over the guard rather than the + // cell. Walking the cell would take the host down, so this is a + // panic-safety fix and not only a correctness one. + // + // Nothing is written back for a shared root: cloning a shared + // `Dynamic` clones the `Rc`, so a mutation through the guard already + // landed in the cell every other holder can see. + let shared = is_shared!(root); + let result = if shared { + let mut guard = root.write_lock::().ok_or_else(|| { + let name = root_name(program, chain).unwrap_or_default(); + Box::new(EvalAltResult::ErrorDataRace(name.to_string(), pos)) + })?; + self.walk_chain( + program, + chain, + &chain.steps, + &mut guard, + &mut operands, + value, + pos, + ) + } else { + self.walk_chain( + program, + chain, + &chain.steps, + &mut root, + &mut operands, + value, + pos, + ) + }; + + // A place is the one root left that writes back — the entry cannot be + // held across the walk without borrowing the scope for its whole + // duration, so the walk gets a copy and this puts it back. + // + // Not a constant, which could not have been changed anyway: the walk + // was handed a read-only value, so anything that would have mutated it + // refused rather than mutating the copy. + // + // Whether the walk *failed* is not part of it. Rhai reaches the entry + // through a live `&mut`, so a step that mutates and then raises has + // already written — `a.push_then_fail()` inside a `try` leaves the + // push. Gating this on success would discard exactly that. + if chain.mutates() && !shared && !root.is_read_only() { + match at { + RootAt::Place(Site::Slot(index)) => *scope.get_mut_by_index(index) = root, + RootAt::Place(Site::Name(name)) => { + let entry = scope + .get_mut(name) + .ok_or_else(|| malformed(format!("`{name}` stopped being writable")))?; + *entry = root; + } + // The register, for the same reason and under the same rule: + // `this.push(1)` has to reach the caller's value, and the + // binder that owns it is what carries it back out of the frame. + RootAt::This => { + if let Some(entry) = self.this.as_mut() { + *entry = root; + } + } + RootAt::Constant | RootAt::Temporary => {} + } + } + + let (out, _) = result?; + self.stack.truncate(operands_at); + Ok(out) + } + + /// Find what a chain is rooted at, resolving a name if that is what it is. + /// + /// The search is `load_named`'s and the order is observable: a resolver + /// registered with `Engine::on_var` sees the name before the scope does, + /// and a name in no scope is looked for among the global modules before it + /// is reported missing. It runs exactly once — a chain is one instruction, + /// so unlike [`Op::CallRef`] there is nothing to resolve twice. + /// + /// `ErrorVariableNotFound` is reported against the *variable*, which is why + /// [`Root::Named`] carries a position of its own. + fn chain_root<'p>( + &mut self, + program: &'p Program, + chain: &Chain, + scope: &mut Scope, + base: usize, + operands_at: usize, + pos: Position, + ) -> Result, Box> { + // The walk gets a copy of the entry, and cloning a `Dynamic` marks the + // copy read-write however the original was — so a constant has to be + // told it came from one. See [`ChainRoot::value`]. + let walkable = |value: &Dynamic| { + if value.is_read_only() { + value.clone().into_read_only() + } else { + value.clone() + } + }; + + match chain.root { + Root::Local { slot, .. } => { + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!( + "chain root slot {index} is out of scope" + ))); + } + Ok(ChainRoot { + at: RootAt::Place(Site::Slot(index)), + value: walkable(scope.get_mut_by_index(index)), + pos, + }) + } + + // The `this` position wins over the chain's for the same reason a + // name's does: this lookup can fail, and rhai blames the `this` + // rather than the `.` after it (`eval/chaining.rs:519-527`). + Root::This { pos: this_pos } => { + let value = self + .this + .as_ref() + .ok_or_else(|| Box::new(EvalAltResult::ErrorUnboundThis(this_pos)))?; + Ok(ChainRoot { + at: RootAt::This, + value: walkable(value), + pos: this_pos, + }) + } + + // A name has a position of its own, and it wins: the lookup below + // can fail, and rhai blames the variable rather than the chain. + Root::Named { name, pos: var_pos } => { + let name = program + .name(name) + .ok_or_else(|| malformed(format!("no name {name}")))?; + + // A resolver hands back a value rather than a place, which is + // what makes writing through it an error. + if let Some(value) = self.resolve_var(name, scope, var_pos)? { + return Ok(ChainRoot { + at: RootAt::Constant, + value: value.into_read_only(), + pos: var_pos, + }); + } + if let Some(value) = scope.get(name) { + return Ok(ChainRoot { + at: RootAt::Place(Site::Name(name)), + value: walkable(value), + pos: var_pos, + }); + } + // A constant a host published with `Module::set_var`. Not + // marked read-only, because rhai does not mark it either + // (`eval/expr.rs:151` against `:122`) — so a chain assigns + // into the copy and discards it, where writing to the name + // directly is refused. + self.engine + .global_modules + .iter() + .find_map(|module| module.get_var(name)) + .map(|value| ChainRoot { + at: RootAt::Constant, + value, + pos: var_pos, + }) + .ok_or_else(|| missing(name, var_pos)) + } + + Root::Temporary => Ok(ChainRoot { + at: RootAt::Temporary, + value: self.stack[operands_at + chain.operands as usize] + .clone() + .flatten(), + pos, + }), + } + } + + /// One level of the walk. Returns the value and whether anything below may + /// have written. + #[allow(clippy::too_many_arguments)] + fn walk_chain( + &mut self, + program: &Program, + chain: &Chain, + steps: &[Step], + target: &mut Dynamic, + operands: &mut [Dynamic], + value: Option, + pos: Position, + ) -> Result<(Dynamic, bool), Box> { + let Some((step, rest)) = steps.split_first() else { + // The end of the chain, reached with nothing to do: a bare `a` is + // not a chain, so this only happens for an empty step list. + return Ok((target.clone(), false)); + }; + let last = rest.is_empty(); + + match step { + Step::Index { + operand, + pos: idx_pos, + bracket, + } => { + let idx = operands + .get_mut(*operand as usize) + .ok_or_else(|| malformed("chain index operand missing".to_string()))?; + let mut idx = idx.clone(); + // Rhai reports an out-of-bounds index against the index and a + // value that cannot be indexed at all against this step's `[`. + // Both belong to the step, and neither is the chain's. + let idx_pos = *idx_pos; + let bracket = *bracket; + + // Split out so the borrow `get_indexed_mut` takes of `target` + // ends when it returns: the fallback below needs `target` + // again, and a `Target` in scope would still be holding it. + match self.index_by_reference( + program, chain, rest, target, &mut idx, idx_pos, operands, value, last, + bracket, pos, + )? { + Indexed::Done(out, changed) => Ok((out, changed)), + Indexed::NoReference(value) => { + self.assign_through_indexer( + program, chain, target, &mut idx, value, bracket, + )?; + Ok((Dynamic::UNIT, true)) + } + } + } + + Step::Property { + name, + getter, + setter, + pos: step_pos, + } => self.walk_property( + program, chain, rest, target, operands, value, pos, *step_pos, *name, *getter, + *setter, + ), + + Step::Method { + name, + argc, + operand, + pos: step_pos, + } => { + let step_pos = *step_pos; + let name_index = *name; + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + let first = *operand as usize; + let argc = *argc as usize; + if first + argc > operands.len() { + return Err(malformed("chain method arguments missing".to_string())); + } + + // A method call is where rhai tries the receiver's type before + // the plain name (`func/call.rs:614-629`), and the only place it + // does. `argc` already excludes the receiver, which is the arity + // the script side is keyed on (`parser.rs:2128-2145`). + // + // Consulting our own table first is safe because nothing can get + // between: `import` pushes onto `global.modules`, not + // `global.lib` (`eval/stmt.rs:947`), and `global.lib` is only + // ever pushed where an AST is being run. + let type_name = target.type_name(); + let compiled = { + let typed = self.engine.map_type_name(type_name); + program + .method(name_index, argc, typed) + .map(|f| (f.params.clone(), f.chunk)) + }; + + let mut args: Vec = operands[first..first + argc].to_vec(); + let out = if let Some((params, chunk)) = compiled { + // The receiver is moved into the frame and moved back, so a + // write through `this` lands here and the chain's own + // write-back carries it the rest of the way. + let (bound, write_back) = bind_this(target); + let at = self.stack.len(); + self.stack.extend(args); + let (result, returned) = self.call_compiled_with_this( + program, + name, + ¶ms, + chunk, + at, + step_pos, + Some(bound), + ); + self.stack.truncate(at); + // Before `?`: a body that mutated and then raised has + // already written, as it would through rhai's pointer. + unbind_this(target, returned, write_back); + result? + } else { + let mut call_args: Vec<&mut Dynamic> = core::iter::once(&mut *target) + .chain(args.iter_mut()) + .collect(); + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, true, true, &mut call_args) + .map_err(|err| dispatch_failure(err, step_pos))? + }; + + if last { + match value { + // `a.f() = x` is not something rhai parses. + Some(_) => Err(malformed("assignment to a method call".to_string())), + None => Ok((out, true)), + } + } else { + let mut inner = out; + let (out, _) = + self.walk_chain(program, chain, rest, &mut inner, operands, value, pos)?; + // Whatever the sub-chain did, it did to the method's + // return value, which nothing owns. + Ok((out, true)) + } + } + } + } + + /// One `[i]` step, taken through a reference into the container. + /// + /// Returns [`Indexed::NoReference`] when there is no reference to be had — + /// a custom indexer being assigned through — handing the value back so the + /// caller can take the long way round once this borrow has ended. + #[allow(clippy::too_many_arguments)] + fn index_by_reference( + &mut self, + program: &Program, + chain: &Chain, + rest: &[Step], + target: &mut Dynamic, + idx: &mut Dynamic, + idx_pos: Position, + operands: &mut [Dynamic], + value: Option, + last: bool, + bracket: Position, + pos: Position, + ) -> Result> { + let assigning = last && value.is_some(); + let mut detached = Scope::new(); + + let mut item = match self.engine.get_indexed_mut( + &mut self.global, + &mut self.caches, + &mut detached, + None, + target, + idx, + idx_pos, + bracket, + // Auto-vivify a missing map key only when writing, as rhai does + // for the assignment case (`eval/chaining.rs:791`). + assigning, + // And do not reach for a custom indexer when writing: a value it + // handed back could not be assigned through. Rhai asks the same + // way and takes the error as its signal. + !assigning, + ) { + Ok(item) => item, + Err(err) if assigning && matches!(*err, EvalAltResult::ErrorIndexingType(..)) => { + return Ok(Indexed::NoReference(value.expect("assigning"))); + } + Err(err) => return Err(err), + }; + + // A read changes nothing, so it consumes the target and there is + // nothing to put back. + if last && value.is_none() { + return Ok(Indexed::Done(item.take_or_clone(), false)); + } + + let temp = item.is_temp_value(); + let (out, changed) = if last { + let value = value.expect("checked above"); + self.store( + program, + chain_op(program, chain)?, + item.as_mut(), + value, + pos, + )?; + (Dynamic::UNIT, true) + } else { + // Straight through the borrow: for an array, a map or a blob this + // *is* the container's element, so a mutation below lands where + // rhai's would. + self.walk_chain(program, chain, rest, item.as_mut(), operands, value, pos)? + }; + + // Bit-fields, string characters and blob bytes cannot be pointed at + // directly, so `Target` carries a copy and this is what puts it back + // (`eval/target.rs:282`). + item.propagate_changed_value(pos)?; + + if temp && changed { + // The element was a temporary — a custom indexer's — so the setter + // is the only way back (`eval/chaining.rs:744`). + let mut updated = item.take_or_clone(); + let mut index = idx.clone(); + self.call_indexer_set(target, &mut index, &mut updated, bracket)?; + } + + Ok(Indexed::Done(out, changed)) + } + + /// Assign through a custom indexer, which cannot hand out a reference. + /// + /// An op-assignment has to read the current value back through the getter + /// first, and rhai *ignores* a getter that fails here — a write-only + /// indexer takes the new value as-is (`eval/chaining.rs:812`). + fn assign_through_indexer( + &mut self, + program: &Program, + chain: &Chain, + target: &mut Dynamic, + index: &mut Dynamic, + value: Dynamic, + pos: Position, + ) -> Result<(), Box> { + let mut new_val = value; + + if matches!(chain.tail, Tail::Assign { op: Some(_) }) { + let mut probe = index.clone(); + if let Ok(mut current) = self.call_indexer(FN_IDX_GET, target, &mut probe, pos) { + self.store( + program, + chain_op(program, chain)?, + &mut current, + new_val, + pos, + )?; + new_val = current; + } + } + + self.call_indexer_set(target, index, &mut new_val, pos) + } + + /// Call the index getter, which unlike the setter is allowed to fail. + fn call_indexer( + &mut self, + name: &str, + target: &mut Dynamic, + index: &mut Dynamic, + pos: Position, + ) -> VmResult { + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, true, false, &mut [target, index]) + .map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos); + } + err + }) + } + + /// Put an element back into a container that had no reference to give. + /// + /// A custom indexer returns a value, so a mutation below it landed in a + /// temporary; this is the replay rhai does at `eval/chaining.rs:744`, + /// including swallowing "this type cannot be indexed" the way it does. + fn call_indexer_set( + &mut self, + target: &mut Dynamic, + index: &mut Dynamic, + value: &mut Dynamic, + pos: Position, + ) -> Result<(), Box> { + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + + match context.call_fn_raw(FN_IDX_SET, true, false, &mut [target, index, value]) { + Ok(_) => Ok(()), + Err(err) if matches!(*err, EvalAltResult::ErrorIndexingType(..)) => Ok(()), + Err(mut err) => { + if err.position().is_none() { + err.set_position(pos); + } + Err(err) + } + } + } + + /// `.name`, which is a key on a map and a getter call on anything else. + /// + /// The distinction is rhai's and it is made at runtime, not at parse time + /// (`eval/chaining.rs:898`). It matters for more than speed: a map hands + /// back a reference, so a mutation below lands in the map, while a getter + /// hands back a value that has to be given to the setter afterwards. + #[allow(clippy::too_many_arguments)] + fn walk_property( + &mut self, + program: &Program, + chain: &Chain, + rest: &[Step], + target: &mut Dynamic, + operands: &mut [Dynamic], + value: Option, + pos: Position, + // The property's own position, which is where rhai blames a getter or + // setter that does not exist (`eval/chaining.rs:1039`). `pos` is the + // chain's, and stays that for everything else. + step_pos: Position, + name: u32, + getter: u32, + setter: u32, + ) -> Result<(Dynamic, bool), Box> { + let last = rest.is_empty(); + let key = program + .name(name) + .ok_or_else(|| malformed(format!("no name {name}")))?; + + if target.is_map() { + let mut map = target + .write_lock::() + .ok_or_else(|| malformed("a map that is not a map".to_string()))?; + + // Only a write creates a key. Rhai passes `add_if_not_found` for + // an assignment (`eval/chaining.rs:930`) and withholds it for a + // read (`:959`) and for a step on the way through (`:1086`), so + // reading `m.absent` must leave `m` alone — otherwise a closure + // holding the map sees a key nobody wrote. + if last { + if let Some(value) = value { + let entry = map.entry(key.into()).or_insert(Dynamic::UNIT); + self.store(program, chain_op(program, chain)?, entry, value, pos)?; + return Ok((Dynamic::UNIT, true)); + } + return match map.get(key) { + Some(entry) => Ok((entry.clone(), false)), + None => self.absent_key(key, step_pos).map(|unit| (unit, false)), + }; + } + + return match map.get_mut(key) { + Some(entry) => self.walk_chain(program, chain, rest, entry, operands, value, pos), + // Rhai walks on into a detached unit, so whatever the rest of + // the chain does to it is discarded (`eval/chaining.rs:211`). + None => { + let mut absent = self.absent_key(key, step_pos)?; + drop(map); + self.walk_chain(program, chain, rest, &mut absent, operands, value, pos) + } + }; + } + + // A host type: getter in, setter out. + let call = |vm: &mut Self, fn_name: u32, args: &mut [&mut Dynamic]| -> VmResult { + let fn_name = program + .name(fn_name) + .ok_or_else(|| malformed(format!("no name {fn_name}")))?; + let mut detached = Scope::new(); + let mut context = EvalContext::new( + vm.engine, + &mut vm.global, + &mut vm.caches, + &mut detached, + None, + ); + context + .call_fn_raw(fn_name, true, true, args) + .map_err(|err| positioned(err, step_pos)) + }; + + if last { + if let Some(value) = value { + // `x.p += 1` has to read `p` back through the getter before it + // can add to it — the setter takes a finished value. + let mut new_val = if matches!(chain.tail, Tail::Assign { op: Some(_) }) { + let mut current = call(self, getter, &mut [target])?; + self.store(program, chain_op(program, chain)?, &mut current, value, pos)?; + current + } else { + value + }; + // A setter's return value is thrown away, as in rhai. + let _ = call(self, setter, &mut [target, &mut new_val])?; + return Ok((Dynamic::UNIT, true)); + } + let out = call(self, getter, &mut [target])?; + return Ok((out, false)); + } + + // A getter returns a value, so the rest of the chain works on a + // temporary. Rhai puts it back through the setter when the sub-chain + // was a method call, and skips the setter otherwise. + let mut temp = call(self, getter, &mut [target])?; + let (out, changed) = + self.walk_chain(program, chain, rest, &mut temp, operands, value, pos)?; + if changed { + let _ = call(self, setter, &mut [target, &mut temp])?; + } + Ok((out, changed)) + } + + /// Store into a slot the walk arrived at, through an operator if there is + /// one. + /// + /// Same resolution order as a plain local assignment, and for the same + /// reason: `x += y` is not `x = x + y` unless nothing implements `+=`. + /// The built-in op-assignment for these operands, if rhai has one. + /// + /// Inlined deliberately: `x += 1` in a loop is entirely this, and routing + /// it through the out-of-line resolution below measured 10% on the + /// tight-loop benchmark. Inlining the *whole* of `store` instead costs + /// more than it saves — it took `branch heavy` from 1.59x to 1.41x — so + /// the split is where the two paths part. + #[inline] + fn store_builtin( + &mut self, + op: &AssignOp, + target: &mut Dynamic, + rhs: &mut Dynamic, + pos: impl Fn() -> Position, + ) -> Option>> { + if !self.engine.fast_operators() { + return None; + } + let (func, need_context) = get_builtin_op_assignment_fn(&op.op_assign, target, rhs)?; + let context = + need_context.then(|| native_context(self.engine, "", None, &self.global, pos())); + Some( + func(context, &mut [target, rhs]) + .map(|_| ()) + .map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos()); + } + err + }), + ) + } + + fn store( + &mut self, + program: &Program, + op: Option<&AssignOp>, + target: &mut Dynamic, + mut rhs: Dynamic, + pos: Position, + ) -> Result<(), Box> { + let Some(op) = op else { + *target = rhs; + return Ok(()); + }; + + if let Some(done) = self.store_builtin(op, target, &mut rhs, || pos) { + return done; + } + + let op_assign_name = program + .name(op.op_assign_name) + .ok_or_else(|| malformed("no op-assign name".to_string()))?; + let op_name = program + .name(op.op_name) + .ok_or_else(|| malformed("no operator name".to_string()))?; + + // The real scope may be borrowed by the target, and dispatch does not + // read it anyway — operators resolve against the engine. + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + + match context.call_fn_raw(op_assign_name, true, false, &mut [target, &mut rhs]) { + Ok(_) => Ok(()), + Err(err) + if matches!(&*err, + EvalAltResult::ErrorFunctionNotFound(name, ..) + if name.starts_with(op_assign_name)) => + { + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + let value = context + .call_fn_raw(op_name, true, false, &mut [&mut *target, &mut rhs]) + .map_err(|err| positioned(err, pos))?; + *target = value; + Ok(()) + } + Err(err) => Err(positioned(err, pos)), + } + } + + /// Read a variable no slot names, the way rhai's `search_scope_only` does + /// (`eval/expr.rs:107-155`). + /// + /// Three places in a fixed order, and the order is observable: a resolver + /// registered with `Engine::on_var` sees the name before the scope does, + /// and a name in no scope is looked for among the global modules before it + /// is reported missing. + /// + /// `flatten` is what the two reads differ by, and only for a scope entry: + /// a value position wants what a shared cell contains, and a capture wants + /// the cell. The other two places can only ever produce a value. + /// + /// Kept out of the dispatch loop for the reason [`Vm::call_compiled`] is. + #[inline(never)] + fn load_named( + &mut self, + name: &str, + scope: &mut Scope, + flatten: bool, + pos: Position, + ) -> VmResult { + // A resolver hands back a value, not a place, so it is read-only — + // which is what makes assigning to one an error. + if let Some(value) = self.resolve_var(name, scope, pos)? { + return Ok(value.into_read_only()); + } + + if let Some(value) = scope.get(name) { + return Ok(if flatten { + value.flatten_clone() + } else { + value.clone() + }); + } + + // A constant a host published with `Module::set_var`. + if let Some(value) = self + .engine + .global_modules + .iter() + .find_map(|module| module.get_var(name)) + { + return Ok(value); + } + + Err(missing(name, pos)) + } + + /// Ask the resolver a host registered with `Engine::on_var`, if there is + /// one. + /// + /// `Ok(None)` covers both "no resolver" and "the resolver declined", which + /// are the same thing to every caller. + fn resolve_var( + &mut self, + name: &str, + scope: &mut Scope, + pos: Position, + ) -> Result, Box> { + // Copied out so the borrow is of the engine rather than of `self`, + // which the context below needs mutably. + let engine = self.engine; + let Some(resolver) = &engine.resolve_var else { + return Ok(None); + }; + + let before = scope.len(); + let context = EvalContext::new(engine, &mut self.global, &mut self.caches, scope, None); + // Index zero: rhai passes the slot its parser resolved, and a name + // that reached here had none. + let resolved = resolver(name, 0, context); + + // A resolver that pushed onto the scope has moved every entry a + // parse-time index named, so rhai stops trusting those from here on. + // Nothing this compiler emits depends on them — its slots are counted + // from a base taken before the run — but a fragment's do. + if scope.len() != before { + self.global.always_search_scope = true; + } + + resolved.map_err(|err| { + if err.position().is_none() { + return reposition(err, pos); + } + err + }) + } + + /// Assign to a variable no slot names. + /// + /// Rhai reaches the target through the same search and then refuses + /// anything that is not a reference it can write through: a value the + /// resolver produced, a module's constant, a `const` entry + /// (`eval/stmt.rs:330-344` and `eval/stmt.rs:118-122`). All three are + /// `ErrorAssignmentToConstant`, so the distinction never reaches a script. + #[inline(never)] + fn assign_named( + &mut self, + program: &Program, + op: Option<&AssignOp>, + name: &str, + rhs: Dynamic, + scope: &mut Scope, + pos: Position, + ) -> Result<(), Box> { + let constant = || { + Box::new(EvalAltResult::ErrorAssignmentToConstant( + name.to_string(), + pos, + )) + }; + + if self.resolve_var(name, scope, pos)?.is_some() { + return Err(constant()); + } + + match scope.is_constant(name) { + Some(true) => return Err(constant()), + Some(false) => {} + // Not a variable at all. A module's is a value rather than a + // place, so writing to one is the same refusal as writing to a + // `const`. + None => { + return Err( + if self + .engine + .global_modules + .iter() + .any(|module| module.get_var(name).is_some()) + { + constant() + } else { + missing(name, pos) + }, + ) + } + } + + let entry = scope + .get_mut(name) + .ok_or_else(|| malformed(format!("`{name}` is in scope but not writable")))?; + let mut target = place(entry, name, pos)?; + + self.store(program, op, &mut target, rhs, pos) + } + + /// Call a function pointer, preferring a chunk we compiled. + /// + /// The pointer sits under its arguments. Rhai's own dispatch would work + /// for all of this, but it cannot reach our chunks — the compiled function + /// table is keyed on names from the pool, and a pointer carries a string — + /// so the name is matched against it first and only the miss goes to + /// `call_raw`. + #[inline(never)] + #[allow(clippy::too_many_arguments)] + fn call_fn_ptr( + &mut self, + program: &Program, + argc: usize, + method: bool, + receiver: Option, + scope: &mut Scope, + frame_base: usize, + pos: Position, + ) -> VmResult { + let base = self + .stack + .len() + .checked_sub(argc + 1) + .ok_or_else(|| malformed("function pointer call is missing its target".into()))?; + let mut at = base; + + // In method position a target that is not a pointer means the *first + // argument* is one and the target is the receiver — `obj.call(f, ..)` + // is how a closure is called against a `this`. Rhai reports the + // mismatch against that argument, not against the target, which is why + // the position moves with it. + let mut receiver_at = None; + if method && !self.stack[at].is::() { + receiver_at = Some(at); + at += 1; + if at >= self.stack.len() { + return Err(self.mismatch::(self.stack[at - 1].type_name(), pos)); + } + } + + let pointer = self.stack[at] + .clone() + .try_cast::() + .ok_or_else(|| self.mismatch::(self.stack[at].type_name(), pos))?; + + let taken = self.stack.len() - at - 1; + let curried = pointer.curry().len(); + // A receiver does not change which function a pointer names, only what + // the callee's `this` is: rhai keys its own script pointers on the + // declared parameter count alone (`types/fn_ptr.rs:422`), and binds the + // receiver alongside them. + let function = program + .function_named(pointer.fn_name(), curried + taken) + .map(|f| (f.params.clone(), f.chunk)); + + // Bound once, whichever path takes it. Curried values are spliced in + // above `at`, so the receiver's index is unaffected either way. + let (mut bound, write_back) = match receiver_at { + Some(index) => { + let (value, write_back) = bind_this(&mut self.stack[index]); + (Some(value), write_back) + } + None => (None, false), + }; + + let outcome = if let Some((params, chunk)) = function { + // Curried arguments go in front of the call's own, which is what + // currying means and where the callee's parameters expect them. + let first = at + 1; + self.stack + .splice(first..first, pointer.curry().iter().cloned()); + + let (result, returned) = self.call_compiled_with_this( + program, + pointer.fn_name(), + ¶ms, + chunk, + first, + pos, + bound.take(), + ); + bound = returned; + result + } else { + // Anything else is rhai's: a native function, a name registered + // elsewhere, or a pointer it built itself. + let mut args: Vec = self.stack.drain(at + 1..).collect(); + let context = native_context(self.engine, pointer.fn_name(), None, &self.global, pos); + pointer + .call_raw(&context, bound.as_mut(), &mut args) + .map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos); + } + err + }) + }; + + // Before `?`, as everywhere else: rhai binds the receiver by reference, + // so a closure that writes and then raises has already written. + if let Some(index) = receiver_at { + unbind_this(&mut self.stack[index], bound, write_back); + if write_back { + let updated = self.stack[index].clone(); + self.return_receiver(program, receiver, updated, scope, frame_base)?; + } + } + + let value = outcome?; + self.stack.truncate(base); + Ok(value) + } + + /// Carry a write through `obj.call(f)`'s `this` back to `obj` itself. + /// + /// Rhai binds the receiver by reference (`func/call.rs:862`), so the write + /// lands in the variable. The operand stack only ever held a copy of it, + /// and this is what puts the copy back where it came from. + /// + /// Nothing to do for a shared receiver: it arrived *as* the cell, so the + /// write already landed where every holder can see it — `run_chain`'s rule + /// for a chain root, and for the same reason. + fn return_receiver( + &mut self, + program: &Program, + receiver: Option, + value: Dynamic, + scope: &mut Scope, + frame_base: usize, + ) -> Result<(), Box> { + if is_shared!(value) || value.is_read_only() { + return Ok(()); + } + + match receiver { + Some(Receiver::Local(slot)) => { + let index = frame_base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + *scope.get_mut_by_index(index) = value; + } + Some(Receiver::Named(var)) => { + let name = program + .name(var) + .ok_or_else(|| malformed(format!("no name {var}")))?; + // A resolver's answer or a module constant has no entry behind + // it, and rhai could not have written through one either. + if let Some(entry) = scope.get_mut(name) { + *entry = value; + } + } + Some(Receiver::This) => { + if let Some(entry) = self.this.as_mut() { + *entry = value; + } + } + // A temporary, which rhai mutates a copy of too. + None => {} + } + Ok(()) + } + + /// Concatenate the segments of an interpolated string, reproducing + /// `eval/expr.rs:280-304`. + /// + /// Every step of it is load-bearing. A **string** segment is written + /// straight out and never reaches dispatch, so a host's `to_string` for + /// strings is not consulted here even though `+` would consult it. + /// Anything else goes through rhai's own rendering, which calls **native** + /// functions only — a script `fn to_string` is invisible to it — and + /// substitutes the mapped type name when the call returns a non-string. + /// The size limit is checked after every segment against the running + /// total, not once at the end. + #[inline(never)] + fn append_segment( + &mut self, + segment: Dynamic, + pos: Position, + ) -> Result<(), Box> { + use core::fmt::Write; + + let mut item = segment.flatten(); + let mut rendered = None; + + // A string is written straight out and never reaches dispatch, so a + // host's `to_string` for strings is not consulted here even though `+` + // would consult it. + if !item.is_string() { + let context = native_context(self.engine, FUNC_TO_STRING, None, &self.global, pos); + rendered = Some(print_with_func(FUNC_TO_STRING, &context, &mut item)); + } + + let mut buffer = self + .stack + .last_mut() + .and_then(|value| value.write_lock::()) + .ok_or_else(|| malformed("interpolation lost its buffer".into()))?; + + // `make_mut` is in place while the buffer is uniquely held, which on + // the operand stack it is — so this is one growing allocation rather + // than one per segment. + match rendered { + Some(text) => write!(buffer.make_mut(), "{text}"), + None => write!(buffer.make_mut(), "{item}"), + } + .expect("writing to a string cannot fail"); + let len = buffer.len(); + drop(buffer); + + // After every segment, against the running total — a script must not + // be able to build a string past `max_string_size` and hand it over + // whole. + #[cfg(not(feature = "unchecked"))] + { + self.engine.throw_on_size((0, 0, len)).map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos); + } + err + }) + } + // `unchecked` removes the limits, and with them the only reason to have + // measured. + #[cfg(feature = "unchecked")] + { + let _ = (len, pos); + Ok(()) + } + } + + /// Start iterating a value, the way rhai's `for` does + /// (`eval/stmt.rs:680-703`). + /// + /// Three places are searched by `TypeId`, in order, and the order is + /// rhai's: the modules in the global namespace, then the imports, then the + /// statically registered sub-modules. Nothing matching is `ErrorFor`. + /// + /// The iterable is flattened first — so iterating a captured array walks a + /// snapshot rather than the shared cell — and is consumed by value, which + /// is why the iterator is built once and held for the life of the loop. + #[inline(never)] + fn iter_init(&mut self, iterable: Dynamic, pos: Position) -> Result<(), Box> { + let iterable = iterable.flatten(); + let type_id = iterable.type_id(); + + let func = self + .engine + .global_modules + .iter() + .find_map(|module| module.get_iter(type_id)); + + // Imported and sub-modules can register iterators too, but neither + // exists to be searched under `no_module`. + #[cfg(not(feature = "no_module"))] + let func = func.or_else(|| self.global.get_iter(type_id)).or_else(|| { + self.engine + .global_sub_modules + .values() + .find_map(|module| module.get_qualified_iter(type_id)) + }); + + let func = func.ok_or_else(|| Box::new(EvalAltResult::ErrorFor(pos)))?; + + self.iterators.push(Iteration { + items: func(iterable), + count: -1, + }); + Ok(()) + } + + /// Call `name` with `argc` arguments sitting contiguously from `first` up. + /// + /// A function this compiler lowered is called directly, with no hash and no + /// module walk: the call site's name index and the function's come from the + /// same pool, so equal names have equal indices. Everything else goes to + /// rhai's dispatch, and resolves exactly as it would in the walker. + fn call_stacked( + &mut self, + program: &Program, + name_index: u32, + argc: usize, + first: usize, + pos: Position, + ) -> VmResult { + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + + if let Some(function) = program.function(name_index, argc) { + return self.call_compiled(program, name, &function.params, function.chunk, first, pos); + } + + // Arguments are already contiguous at the top of the operand stack, + // which is exactly the shape rhai's ABI wants (`func/call.rs:36`). It + // consumes them, replacing each with unit, so the caller truncates + // afterwards rather than reusing them. + let mut args: Vec<&mut Dynamic> = self.stack[first..].iter_mut().collect(); + + // A scope of the callee's own, because the scope an `EvalContext` + // carries is the one a *script* function's body runs in + // (`func/call.rs:639`), and rhai passes `None` there (`:1476`). + // Handing over this frame's would let such a body read the caller's + // locals — reachable, because the functions this compiler skips are + // exactly the ones rhai can still find in `global.lib`. + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, false, false, &mut args) + .map_err(|err| dispatch_failure(err, pos)) + } + + /// The same call, with a variable as its first argument and rhai's + /// method-call rewrite applied to it (`func/call.rs:1434-1460`). + /// + /// The other arguments are already on the operand stack and were evaluated + /// before the receiver was reached, which is the order rhai uses and is + /// observable whenever one of them writes to the receiver. + #[inline(never)] + #[allow(clippy::too_many_arguments)] + fn call_by_reference( + &mut self, + program: &Program, + name_index: u32, + argc: usize, + receiver: Receiver, + scope: &mut Scope, + base: usize, + pos: Position, + ) -> VmResult { + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + + // Every argument count here includes the receiver, so zero of them + // names no receiver at all and the instruction is nonsense. Only an + // artifact can say it; the compiler emits one of these for a call that + // has a first argument. + if argc == 0 { + return Err(malformed( + "a call by reference with no receiver".to_string(), + )); + } + + // The register is not a scope entry, so it takes a path of its own + // rather than a third [`Site`]. + if let Receiver::This = receiver { + return self.call_by_this(program, name_index, name, argc, pos); + } + + // A named receiver's value is already argument zero — [`Op::LoadNamed`] + // put it there. A local's is not on the stack at all. + let (at, on_stack) = match receiver { + Receiver::Local(slot) => { + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + (Site::Slot(index), argc - 1) + } + Receiver::Named(var) => { + let name = program + .name(var) + .ok_or_else(|| malformed(format!("no name {var}")))?; + (Site::Name(name), argc) + } + Receiver::This => unreachable!("taken above"), + }; + let first = self + .stack + .len() + .checked_sub(on_stack) + .ok_or_else(|| malformed("call with too few arguments".to_string()))?; + + let place = match at { + Site::Slot(index) => Some(scope.get_mut_by_index(index)), + // A resolver's answer shadows the scope, and `load_named` marks one + // read-only precisely because it is a value and not a place. Asking + // the resolver again to find that out would run it twice, which a + // host can see. + Site::Name(..) if self.stack[first].is_read_only() => None, + Site::Name(name) => scope.get_mut(name), + }; + + // Three things rule out a reference, and rhai rules out the same three: + // it hands one out for neither a shared cell nor a constant + // (`func/call.rs:1449-1454`), and a function this compiler lowered + // copies its first argument whatever it is handed, exactly as rhai + // copies it before running a script function (`func/call.rs:661`). + let by_reference = place + .map(|value| !is_shared!(value) && !value.is_read_only()) + .unwrap_or(false) + && program.function(name_index, argc).is_none(); + + // All three want the ordinary shape, with every argument on the stack. + if !by_reference { + // A local's value has not been pushed. A name's already is: it is + // what carried the lookup's position (see [`Receiver::Named`]), and + // it is exactly the value rhai would pass. + if let Site::Slot(index) = at { + let value = scope.get_mut_by_index(index).flatten_clone(); + self.stack.insert(first, value); + } + let value = self.call_stacked(program, name_index, argc, first, pos); + self.stack.truncate(first); + return value; + } + + let value = { + let (entry, rest) = match at { + Site::Slot(index) => (scope.get_mut_by_index(index), first), + // Argument zero is dead weight now that there is an entry to + // reach, and it is the price of having resolved the name where + // its position was. + Site::Name(name) => ( + scope + .get_mut(name) + .ok_or_else(|| malformed(format!("`{name}` stopped being writable")))?, + first + 1, + ), + }; + let mut args: Vec<&mut Dynamic> = core::iter::once(entry) + .chain(self.stack[rest..].iter_mut()) + .collect(); + // The scope a dispatched script function runs in, which is never + // this frame's — see [`Vm::call_stacked`], which has to build one + // for the same reason and cannot borrow this one because the + // receiver is holding it. + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, true, false, &mut args) + .map_err(|err| dispatch_failure(err, pos)) + }; + + self.stack.truncate(first); + value + } + + /// The same again, with `this` as the first argument. + /// + /// [`Op::LoadThis`] has already pushed a flattened snapshot as argument + /// zero — *before* the other arguments, unlike either of the other two + /// receivers, because the path a shared or unbound receiver takes reads + /// `this` first (`func/call.rs:1462`) where the by-reference path takes a + /// pointer to it afterwards (`:1417`). Reading first is what makes an + /// unbound `f(this, nosuch)` report `ErrorUnboundThis` rather than the + /// argument's failure. + /// + /// The snapshot is what gets passed when the register cannot be lent out, + /// and dead weight when it can — the trade [`Receiver::Named`] makes too. + #[inline(never)] + fn call_by_this( + &mut self, + program: &Program, + name_index: u32, + name: &str, + argc: usize, + pos: Position, + ) -> VmResult { + let first = self + .stack + .len() + .checked_sub(argc) + .ok_or_else(|| malformed("call with too few arguments".to_string()))?; + + // Rhai turns `f(this, ..)` into `this.f(..)` for a receiver that is not + // shared, and read-only is *not* part of that test — unlike the variable + // arm, which copies a constant before deciding (`func/call.rs:1449`). + // A function this compiler lowered copies its first argument whatever it + // is handed, exactly as rhai copies one before running a script function + // (`func/call.rs:661`), so a compiled callee rules a reference out too. + let by_reference = self.this.as_ref().map_or(false, |value| !is_shared!(value)) + && program.function(name_index, argc).is_none(); + + if !by_reference { + let value = self.call_stacked(program, name_index, argc, first, pos); + self.stack.truncate(first); + return value; + } + + let value = { + let entry = self + .this + .as_mut() + .ok_or_else(|| malformed("`this` stopped being bound".to_string()))?; + // Argument zero is the snapshot, dead now that there is a register + // to reach through. + let mut args: Vec<&mut Dynamic> = core::iter::once(entry) + .chain(self.stack[first + 1..].iter_mut()) + .collect(); + // A scope of the callee's own, for [`Vm::call_stacked`]'s reason. + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, true, false, &mut args) + .map_err(|err| dispatch_failure(err, pos)) + }; + + self.stack.truncate(first); + value + } + + fn call_compiled( + &mut self, + program: &Program, + name: &str, + params: &[u32], + chunk: crate::grain::bytecode::Chunk, + first: usize, + pos: Position, + ) -> VmResult { + self.call_compiled_with_this(program, name, params, chunk, first, pos, None) + .0 + } + + /// The same, against a receiver the callee owns for the duration. + /// + /// Hands the receiver back however the call ended, so a body that mutated + /// `this` and then raised still gives its binder something to write back — + /// which is what rhai's pointer into the caller's storage does for free. + /// + /// Every compiled call comes through here, and [`Vm::call_compiled`] is this + /// with no receiver. That is what makes `this` per-call rather than + /// inherited: an ordinary call installs `None` and gives the caller's back + /// on the way out, so a callee can never read the receiver of the frame that + /// called it (`func/call.rs:669`). + /// + /// Kept out of the dispatch loop. Inlined, it is enough extra code to change + /// register allocation across every other instruction — measured as a + /// uniform slowdown on benchmarks that call no functions at all. + #[inline(never)] + fn call_compiled_with_this( + &mut self, + program: &Program, + name: &str, + params: &[u32], + chunk: crate::grain::bytecode::Chunk, + first: usize, + pos: Position, + this: Option, + ) -> (VmResult, Option) { + let saved = mem::replace(&mut self.this, this); + + let result = match self.engine.track_operation(&mut self.global, pos) { + Ok(()) => { + self.global.level += 1; + let result = self.call_compiled_body(program, name, params, chunk, first, pos); + self.global.level -= 1; + result + } + Err(err) => Err(err), + }; + + (result, mem::replace(&mut self.this, saved)) + } + + fn call_compiled_body( + &mut self, + program: &Program, + name: &str, + params: &[u32], + chunk: crate::grain::bytecode::Chunk, + first: usize, + pos: Position, + ) -> VmResult { + #[cfg(not(feature = "unchecked"))] + { + if self.global.level > self.engine.max_call_levels() { + return Err(Box::new(EvalAltResult::ErrorStackOverflow(pos))); + } + if params.len() > self.engine.max_variables() { + return Err(Box::new(EvalAltResult::ErrorTooManyVariables(pos))); + } + } + + // A fresh scope: a function sees its parameters and nothing else. + let mut frame = Scope::new(); + for (param, slot) in params.iter().zip(first..) { + let name = program + .name(*param) + .ok_or_else(|| malformed(format!("no name {param}")))?; + // Taken, not cloned — rhai consumes the caller's argument slots + // (`func/script.rs:75`), and the caller truncates them away after. + let value = self + .stack + .get_mut(slot) + .ok_or_else(|| malformed("call with too few arguments".to_string()))? + .take(); + frame.push_dynamic(name, value); + } + + // A function's parameters are its first locals, sitting at 0 upwards in + // a scope that holds nothing else — so slot 0 is index 0. + let mut reached = chunk.entry() as usize; + let outcome = self.execute(program, &mut frame, chunk, 0, &mut reached); + if outcome.is_err() { + self.fault_pc = Some(reached); + } + + outcome.or_else(|err| match *err { + // A `return` inside the body is the body's value. + EvalAltResult::Return(value, ..) => Ok(value), + // Exits and system errors pass straight through, positioned at the + // call rather than at whatever raised them. + EvalAltResult::Exit(..) => Err(reposition(err, pos)), + _ if err.is_system_exception() => Err(reposition(err, pos)), + // Everything else is attributed to the call. + _ => Err(Box::new(EvalAltResult::ErrorInFunctionCall( + name.to_string(), + self.global.source().unwrap_or("").to_string(), + err, + pos, + ))), + }) + } + + /// Run one frame, cleaning up after it however it leaves. + /// + /// Whatever the frame's loops are holding goes when the frame does — a + /// `return` out of a `for`, or an error escaping one, both skip the + /// `IterNext` that would have dropped the iterator. Doing it here rather + /// than at each exit means there is one place to be right. + fn execute( + &mut self, + program: &Program, + scope: &mut Scope, + chunk: crate::grain::bytecode::Chunk, + base: usize, + reached: &mut usize, + ) -> VmResult { + let iter_base = self.iterators.len(); + let handler_base = self.handlers.len(); + let size_base = self.sizes.len(); + // Each frame's floor is its own. A checkpoint inside a function this + // one calls must not become what this one unwinds to. + let outer_floor = mem::replace(&mut self.unwind_floor, base); + + // The dispatch loop uses `?` throughout, so an error leaves it rather + // than being examined inside it. Catching therefore happens out here: + // the loop stops, a handler this frame armed gets the error, and the + // loop restarts at the catch block. `run_frame` keeps `pc` in a + // register and the fault address arrives through `reached`, which is + // written every instruction anyway, so none of this costs the common + // path anything. + let mut start = chunk.entry() as usize; + let result = loop { + match self.run_frame(program, scope, base, reached, start) { + Ok(value) => break Ok(value), + Err(err) => match self.catch(program, err, handler_base, scope) { + // Metered like a backward jump, and for the same reason: + // a catch block that sits before the throw is a cycle the + // dispatch loop never sees as one, because control got + // there through the error path rather than through a jump. + Ok(resume) => { + self.engine + .track_operation(&mut self.global, program.position(resume))?; + start = resume; + } + Err(err) => break Err(err), + }, + } + }; + + self.iterators.truncate(iter_base); + self.handlers.truncate(handler_base); + self.sizes.truncate(size_base); + + if result.is_err() { + self.unwind_after_error(scope); + } + self.unwind_floor = outer_floor; + result + } + + /// Rhai's `Engine::make_type_mismatch_err` (`api/formatting.rs:246`). + /// + /// The asymmetry is rhai's and is easy to get wrong in either direction: + /// the *expected* type goes through the engine's registered names and the + /// *actual* one does not. So `if 0..1 {}` reports + /// `core::ops::range::Range` rather than the `range` the same engine + /// would print anywhere else. Mapping both — which reads like the obvious + /// thing — makes every one of these differ from the walker. + fn mismatch(&self, actual: &str, pos: Position) -> Box { + Box::new(EvalAltResult::ErrorMismatchDataType( + self.engine + .map_type_name(core::any::type_name::()) + .into(), + actual.into(), + pos, + )) + } + + /// What reading a key a map does not have produces. + /// + /// Unit, unless the host asked for the strict reading — which is a whole + /// engine option (`fail_on_invalid_map_property`) rather than anything the + /// script says, so it has to be consulted rather than assumed. + fn absent_key(&self, key: &str, pos: Position) -> VmResult { + if self.engine.fail_on_invalid_map_property() { + Err(Box::new(EvalAltResult::ErrorPropertyNotFound( + key.to_string(), + pos, + ))) + } else { + Ok(Dynamic::UNIT) + } + } + + /// Fold the element on top of the stack into the literal's running total, + /// and refuse it if that puts the literal over a configured limit. + /// + /// Reproduces `eval/expr.rs:318-329` for an array and `:349-359` for a map. + /// The two differ in one place — an array element adds one to the array + /// count, a map entry adds one to the map count — and in nothing else, so + /// they share this. + /// + /// Worth being exact about what is *not* counted: a map's total starts at + /// zero and only the entries with computed values are added to it, because + /// rhai's loop runs over those alone and the constant ones are already + /// sitting in the template. A literal that is entirely constant never + /// reaches here at all — the optimizer folded it long before. + /// + /// Out of line: it is a handful of instructions in the common case and a + /// call to rhai in the rare one, and the dispatch loop is measurably + /// sensitive to what shares its registers. + #[inline(never)] + fn check_size( + &mut self, + index: u16, + map: bool, + pos: Position, + ) -> Result<(), Box> { + if index == 0 { + self.sizes.push((0, 0, 0)); + } + + // `unchecked` removes every limit this could reject against, so the + // running total is never read and measuring it is pure cost. The push + // above still happens: the stack is frame-floored either way, and the + // instruction that drops it does not know which build it is in. + #[cfg(feature = "unchecked")] + { + let _ = (map, pos); + Ok(()) + } + + #[cfg(not(feature = "unchecked"))] + { + // Rhai skips the whole measurement when no limit could reject it, + // and measuring is a walk of the value — so this is the difference + // between free and proportional to what the literal holds. + if self.engine.max_string_size() == 0 + && self.engine.max_array_size() == 0 + && self.engine.max_map_size() == 0 + { + return Ok(()); + } + + let value = self + .stack + .last() + .ok_or_else(|| malformed("size check with no element".to_string()))?; + let delta = calc_data_sizes(value, true); + + let total = self + .sizes + .last_mut() + .ok_or_else(|| malformed("size check outside a literal".to_string()))?; + *total = ( + total.0 + delta.0 + usize::from(!map), + total.1 + delta.1 + usize::from(map), + total.2 + delta.2, + ); + + self.engine + .throw_on_size(*total) + .map_err(|err| positioned(err, pos)) + } + } + + /// Drop what an escaping error skipped the unwind for. + /// + /// An error leaves a block by jumping over the [`Op::UnwindTo`] that would + /// have dropped what it declared, so those locals are still in the scope. + /// Rhai rewinds a block whether it is left normally or by a throw, and + /// rewinds nothing at a chunk's top level — which is what the floor is: the + /// last top-level statement boundary. Anything above it belongs to a block + /// that did not get to finish. + /// + /// Guarded rather than unconditional because [`Op::Return`] has already + /// unwound to `base`, which is below the floor. + /// + /// Out of line for the reason [`Vm::catch`] is: it sits on the error edge + /// of the frame, where nothing is hot and everything competes with the + /// dispatch loop for the same registers. + #[inline(never)] + fn unwind_after_error(&self, scope: &mut Scope) { + if scope.len() > self.unwind_floor { + scope.rewind(self.unwind_floor); + } + } + + /// Hand an error to the innermost handler this frame armed, if any. + /// + /// `Ok` is the address the catch block starts at. `Err` means nothing here + /// wanted it and it should keep going up. + /// + /// Kept out of line for the reason [`Vm::call_compiled`] is: it sits on + /// the dispatch loop's error edge, and letting it inline there costs every + /// instruction that never fails. + #[inline(never)] + fn catch( + &mut self, + program: &Program, + err: Box, + handler_base: usize, + scope: &mut Scope, + ) -> Result> { + // Only handlers this frame armed. A callee must never resume into its + // caller's catch block — that is a jump into another chunk, which the + // verifier forbids and nothing would catch at run time. The callee's + // error propagates normally instead, and `ErrorInFunctionCall` is + // catchable, so the caller's own frame still sees it. + // Walking outwards, because leaving one region can land the error in + // the next: `try { try { throw 1 } catch { throw; } } catch (e) { .. }` + // re-raises from the inner catch and the outer `try` still has to see + // it. + let mut err = err; + let handler = loop { + if self.handlers.len() <= handler_base { + return Err(err); + } + let handler = self.handlers.last_mut().expect("checked"); + + // Leaving a catch block rather than entering one. A bare `throw;` + // there — an `ErrorRuntime` carrying unit — means "re-raise what + // was caught, from here" (`eval/stmt.rs:866`). + let Some(original) = handler.caught.take() else { + break handler; + }; + self.handlers.pop(); + let rethrown = + matches!(&*err, EvalAltResult::ErrorRuntime(value, ..) if value.is_unit()); + if rethrown { + let pos = err.position(); + err = original; + err.set_position(pos); + } + }; + + // `return`, `break`, `continue`, `exit` and the system exceptions + // unwind as errors and are not exceptions a script may catch. + if !err.is_catchable() { + return Err(err); + } + + let (target, catch_var) = (handler.target, handler.catch_var); + let (operands, scope_len, iters) = (handler.operands, handler.scope_len, handler.iters); + + let mut err = err; + let value = self.catch_value(&mut err, catch_var.is_some()); + + // Back to where the `try` began, at all three depths. + self.stack.truncate(operands); + self.iterators.truncate(iters); + scope.rewind(scope_len); + + if let Some(index) = catch_var { + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + #[cfg(not(feature = "unchecked"))] + if scope.len() >= self.engine.max_variables() { + return Err(Box::new(EvalAltResult::ErrorTooManyVariables( + program.position(target), + ))); + } + scope.push_dynamic(name, value); + } + + self.handlers.last_mut().expect("checked").caught = Some(err); + Ok(target) + } + + /// What the catch variable is bound to (`eval/stmt.rs:809-845`). + /// + /// Three shapes: nothing at all without a variable, the raw thrown value + /// for a `throw`, and a map of the error's parts for anything else. The + /// unwrapping matters — a `throw` inside a called function arrives wrapped + /// in `ErrorInFunctionCall`, and rhai still binds the bare value. + fn catch_value(&self, err: &mut Box, wanted: bool) -> Dynamic { + if !wanted { + return Dynamic::UNIT; + } + if let EvalAltResult::ErrorRuntime(value, ..) = err.unwrap_inner() { + return value.clone(); + } + + let mut map = Map::new(); + // Read *and cleared*, as rhai does, so the message below carries no + // trailing position and a re-raise starts from the catch site. + let pos = err.take_position(); + + map.insert("message".into(), err.to_string().into()); + if let Some(source) = &self.global.source { + map.insert("source".into(), source.into()); + } + if !pos.is_none() { + let line = pos.line().unwrap_or(0) as INT; + map.insert("line".into(), line.into()); + let column = pos.position().unwrap_or(0) as INT; + map.insert("position".into(), column.into()); + } + err.dump_fields(&mut map); + map.into() + } + + /// The dispatch loop. `start` is the chunk's entry, or a catch block's + /// address when [`Vm::execute`] resumes one after an error. + /// + /// Inlined into its one caller: splitting the loop out so errors could be + /// caught outside it cost 1.55x to 1.40x on the tight-loop benchmark until + /// this was here. + #[inline(always)] + fn run_frame( + &mut self, + program: &Program, + scope: &mut Scope, + base: usize, + reached: &mut usize, + start: usize, + ) -> VmResult { + // A called function pushes its operands above the caller's rather than + // starting a stack of its own, so this records where its own begin. + let stack_base = self.stack.len(); + self.stack.reserve(program.max_stack() as usize); + + // A residual's `Expr::Variable` nodes carry offsets rhai's parser + // computed against its own scope discipline, not against ours. Forcing + // name lookup inside them costs a reverse scan but cannot be wrong. + // Only programs that still have residuals pay it, which is the point of + // driving the count to zero. + if program.residual_count() > 0 { + self.global.always_search_scope = true; + } + + let code = program.code(); + // The chunk's entry the first time round, a catch block's address when + // resumed after one. + let mut pc = start; + + loop { + // Nothing inside an iteration moves `pc` except a jump, and a jump + // only happens after the instruction succeeded — so recording it + // here names whichever instruction fails. + *reached = pc; + + // No check against the chunk's end. Verification proves execution + // cannot leave it — every path reaches a `Return`, no jump goes + // outside, nothing falls off — so a comparison here would cost + // every instruction to restate something already established. + let tag = *code.get(pc).ok_or_else(|| { + Box::new(EvalAltResult::ErrorRuntime( + format!("ran off the end of a chunk at {pc}").into(), + Position::NONE, + )) + })?; + + // Every instruction's operands sit at a fixed offset from its tag, + // so dispatch is a match and a couple of loads with nothing decoded + // and nothing allocated. The bounds checks are what let this run + // straight off an artifact without trusting it; the verifier has + // already made them unreachable for anything that loaded. + let width = code::width(code, pc) + .ok_or_else(|| malformed(format!("undecodable instruction at {pc}")))?; + let small = |offset: usize| { + code::u16_at(code, pc + offset) + .ok_or_else(|| malformed(format!("truncated operand at {pc}"))) + }; + let wide = |offset: usize| { + code::u32_at(code, pc + offset) + .ok_or_else(|| malformed(format!("truncated operand at {pc}"))) + }; + + // Instructions carry no position; the table does, keyed on the + // address. A stripped program answers `NONE` for every one of + // these, which is what a device runs — the address travels back + // with the error instead, and the host resolves it. + // + // A closure rather than a value: most instructions never ask, and + // the ones that do mostly ask only on the way to an error. + let pos = || program.position(pc); + + // Every transfer of control goes through this, and a backward one + // is charged an operation. + // + // A cycle in a chunk always contains a backward edge, so this is + // what makes `max_operations` and the `on_progress` interrupt cover + // a chunk *this compiler did not write*. `Op::Tick` covers the + // loops it does write, positioned where rhai would report them; a + // corrupt artifact has no ticks at all and would otherwise spin + // forever inside a loader that had already accepted it. Found by + // `mutated_artifacts_load_or_fail_but_never_misbehave`, whose whole + // claim is that this cannot happen. + // + // A macro rather than four open-coded checks because the failure + // mode of missing one is silent, and because it costs nothing on + // the straight-line path: only a jump pays the comparison. + macro_rules! transfer { + ($target:expr) => {{ + let target: usize = $target; + if target <= pc { + self.engine.track_operation(&mut self.global, pos())?; + } + pc = target; + }}; + } + + match tag { + code::tag::CONST => { + let index = u32::from(small(1)?); + let value = program + .constant(index) + .ok_or_else(|| malformed(format!("no constant {index}")))?; + self.stack.push(value.clone()); + } + + code::tag::UNIT => self.stack.push(Dynamic::UNIT), + code::tag::FALSE => self.stack.push(Dynamic::from(false)), + code::tag::TRUE => self.stack.push(Dynamic::from(true)), + + code::tag::LOAD_LOCAL => { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + // Reads clone out, matching how rhai's own variable reads + // leave the scope entry alone (`eval/expr.rs:276-278`), and + // flattening any shared cell the way a read should. + self.stack + .push(scope.get_mut_by_index(index).flatten_clone()); + } + + code::tag::STORE_LOCAL => { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + let value = self.pop()?; + // Through the cell, not over it — see `place`. + *place(scope.get_mut_by_index(index), "", pos())? = value; + } + + code::tag::LOAD_NAMED | code::tag::LOAD_SHARED_NAMED => { + let index = u32::from(small(1)?); + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + let flatten = tag == code::tag::LOAD_NAMED; + let value = self.load_named(name, scope, flatten, pos())?; + self.stack.push(value); + } + + code::tag::ASSIGN_NAMED | code::tag::ASSIGN_NAMED_OP => { + let index = u32::from(small(1)?); + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + let op = if tag == code::tag::ASSIGN_NAMED_OP { + let index = u32::from(small(3)?); + Some( + program + .assign_op(index) + .ok_or_else(|| malformed(format!("no op-assignment {index}")))?, + ) + } else { + None + }; + + // Flattened before assigning, as rhai does, so a shared + // cell is copied out rather than aliased into the target. + let rhs = self.pop()?.flatten(); + self.assign_named(program, op, name, rhs, scope, pos())?; + } + + code::tag::DECLARE_LOCAL | code::tag::DECLARE_CONST => { + let index = u32::from(small(1)?); + // A `Scope` entry name is an `Identifier`, which is a + // `SmartString` — short names live inline, so handing it a + // borrowed `&str` costs a copy rather than an allocation. + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + let value = self.pop()?; + if tag == code::tag::DECLARE_CONST { + scope.push_constant_dynamic(name, value); + } else { + scope.push_dynamic(name, value); + } + } + + code::tag::ASSIGN_LOCAL | code::tag::ASSIGN_LOCAL_OP => { + let slot = small(1)?; + let var_name = u32::from(small(3)?); + let op = if tag == code::tag::ASSIGN_LOCAL_OP { + let index = u32::from(small(5)?); + Some( + program + .assign_op(index) + .ok_or_else(|| malformed(format!("no op-assignment {index}")))?, + ) + } else { + None + }; + + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + + // Rhai flattens the right-hand side before assigning + // (`eval/stmt.rs:324`), so a shared cell is copied out + // rather than aliased into the target. + let rhs = self.pop()?.flatten(); + + if scope.get_mut_by_index(index).is_read_only() { + let name = program + .name(var_name) + .ok_or_else(|| malformed(format!("no name {var_name}")))?; + return Err(Box::new(EvalAltResult::ErrorAssignmentToConstant( + name.to_string(), + pos(), + ))); + } + + // Written through rather than over: a slot a closure + // captured is a shared cell, and replacing it would sever + // every holder. `store` is the same path a chain's tail + // and a named assignment take, so `x op= y` resolves + // identically wherever the target lives. + let name = program + .name(var_name) + .ok_or_else(|| malformed(format!("no name {var_name}")))?; + // The guard is only needed for a cell a closure captured, + // and `x += 1` in a loop is the hot path — so the check + // for one is a discriminant test rather than the downcast + // chain `write_lock` walks, and the built-in operator is + // reached without leaving the dispatch loop or resolving + // the position. + // + // It is not free even so: the tight-loop benchmark went + // 1.63x to 1.55x when locals stopped being written over + // and started being written through. That is the price of + // a shared cell surviving an assignment, and of a chain + // over one not taking the host down. + let entry = scope.get_mut_by_index(index); + if !is_shared!(entry) { + let mut rhs = rhs; + if let Some(done) = + op.and_then(|op| self.store_builtin(op, entry, &mut rhs, pos)) + { + done?; + pc += width; + continue; + } + self.store(program, op, entry, rhs, pos())?; + pc += width; + continue; + } + + let mut target = place(entry, name, pos())?; + self.store(program, op, &mut target, rhs, pos())?; + } + + code::tag::LOAD_THIS | code::tag::LOAD_THIS_SHARED => { + let value = self + .this + .as_ref() + .ok_or_else(|| Box::new(EvalAltResult::ErrorUnboundThis(pos())))?; + // Rhai's read is `this_ptr.cloned()` and does not flatten + // (`eval/expr.rs:272`); its consumers do. Which tag this is + // is which consumer asked. + self.stack.push(if tag == code::tag::LOAD_THIS { + value.flatten_clone() + } else { + value.clone() + }); + } + + code::tag::REQUIRE_THIS => { + if self.this.is_none() { + return Err(Box::new(EvalAltResult::ErrorUnboundThis(pos()))); + } + } + + code::tag::ASSIGN_THIS | code::tag::ASSIGN_THIS_OP => { + let op = if tag == code::tag::ASSIGN_THIS_OP { + let index = u32::from(small(1)?); + Some( + program + .assign_op(index) + .ok_or_else(|| malformed(format!("no op-assignment {index}")))?, + ) + } else { + None + }; + + // Flattened before assigning, as everywhere else. + let rhs = self.pop()?.flatten(); + + // Taken out of the register rather than borrowed from it: + // `store` wants the whole `Vm`, and a write lock into the + // field could not outlive that borrow. Put back on both + // paths — rhai's mutation survives an error, and a frame + // that lost its receiver would answer `ErrorUnboundThis` to + // every read after this one. + let mut this = self + .this + .take() + .ok_or_else(|| Box::new(EvalAltResult::ErrorUnboundThis(pos())))?; + + let outcome = if this.is_read_only() { + // Named for an expression that has no name, which is + // what rhai reports too (`eval/stmt.rs:118-122`). + Err(Box::new(EvalAltResult::ErrorAssignmentToConstant( + String::new(), + pos(), + ))) + } else { + // Written through, not over: a shared receiver has to + // keep its cell, as a captured local does. + match place(&mut this, "", pos()) { + Ok(mut target) => self.store(program, op, &mut target, rhs, pos()), + Err(err) => Err(err), + } + }; + + self.this = Some(this); + outcome?; + } + + code::tag::POP => { + let _ = self.pop()?; + } + + code::tag::EVAL_AST | code::tag::EVAL_AST_KEEP => { + let index = u32::from(small(1)?); + let expr = program + .residual(index) + .ok_or_else(|| malformed(format!("no residual {index}")))?; + let rewind_scope = tag == code::tag::EVAL_AST; + + // Straight to the walker's own entry points rather than + // through `EvalContext::eval_expression_tree_raw`, which + // is the same two calls behind a shim that only exists + // under `custom_syntax`. Total language coverage rests on + // this, so it must not depend on a feature. + // + // The frame's receiver goes with it, by reference. A body + // that uses `this` can still hold a fragment — `this?.x`, + // or a `this` body containing an `import` — and the walker + // has to read and write the same receiver the surrounding + // instructions do. The engine is copied out first so the + // four borrows below are of disjoint fields. + let engine = self.engine; + let value = match expr { + Expr::Stmt(block) => engine.eval_stmt_block( + &mut self.global, + &mut self.caches, + scope, + self.this.as_mut(), + block.statements(), + rewind_scope, + ), + expr => engine.eval_expr( + &mut self.global, + &mut self.caches, + scope, + self.this.as_mut(), + expr, + ), + }?; + + self.stack.push(value); + } + + code::tag::JUMP => { + transfer!(wide(1)? as usize); + continue; + } + + code::tag::JUMP_IF_FALSE | code::tag::JUMP_IF_TRUE => { + let target = wide(1)? as usize; + let condition = self.pop()?; + // Rhai requires a boolean guard and reports the mismatch at + // the guard's own position (`eval/stmt.rs:487-490`). + let holds = condition + .as_bool() + .map_err(|actual| self.mismatch::(actual, pos()))?; + if holds == (tag == code::tag::JUMP_IF_TRUE) { + transfer!(target); + continue; + } + } + + code::tag::CALL | code::tag::CALL_OP => { + let name_index = u32::from(small(1)?); + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + let argc = code[pc + 3] as usize; + let op = if tag == code::tag::CALL_OP { + let index = u32::from(small(4)?); + Some( + program + .token(index) + .ok_or_else(|| malformed(format!("no operator {index}")))?, + ) + } else { + None + }; + + let first = self + .stack + .len() + .checked_sub(argc) + .ok_or_else(|| malformed("call with too few arguments".to_string()))?; + + // Reach the same built-in the walker reaches. Gated on + // rhai's own `fast_operators()` rather than a guard of our + // own, so an engine that turns it off gets the dispatch + // path on both sides, and one that leaves it on gets the + // same answer — including for a user-registered operator + // on a primitive, which rhai's fast path also bypasses + // (`func/call.rs:1775-1799`). + if let (Some(token), 2, true) = (op, argc, self.engine.fast_operators()) { + let (lhs, rhs) = self.stack.split_at_mut(first + 1); + let lhs = &mut lhs[first]; + let rhs = &mut rhs[0]; + + // Custom types go to dispatch first, so a registered + // function still wins for them. + let builtin = (!lhs.is_variant() && !rhs.is_variant()) + .then(|| get_builtin_binary_op_fn(token, lhs, rhs)) + .flatten(); + if let Some((func, need_context)) = builtin { + let context = need_context.then(|| { + native_context(self.engine, name, None, &self.global, pos()) + }); + let value = func(context, &mut [lhs, rhs])?; + self.stack.truncate(first); + self.stack.push(value); + pc += width; + continue; + } + } + + let value = self.call_stacked(program, name_index, argc, first, pos())?; + self.stack.truncate(first); + self.stack.push(value); + } + + code::tag::CALL_LOCAL_REF + | code::tag::CALL_NAMED_REF + | code::tag::CALL_THIS_REF => { + let name_index = u32::from(small(1)?); + let argc = code[pc + 3] as usize; + // `this` is a register, so this one carries no operand for + // the receiver and is two bytes shorter. + let receiver = match tag { + code::tag::CALL_LOCAL_REF => Receiver::Local(small(4)?), + code::tag::CALL_NAMED_REF => Receiver::Named(u32::from(small(4)?)), + _ => Receiver::This, + }; + + let value = self.call_by_reference( + program, + name_index, + argc, + receiver, + scope, + base, + pos(), + )?; + self.stack.push(value); + } + + code::tag::ROTATE => { + let under = code[pc + 1] as usize; + let top = self + .stack + .len() + .checked_sub(1) + .ok_or_else(|| malformed("rotate on an empty stack".to_string()))?; + let to = top + .checked_sub(under) + .ok_or_else(|| malformed("rotate past the bottom".to_string()))?; + self.stack[to..].rotate_right(1); + } + + code::tag::MAKE_ARRAY => { + let len = small(1)? as usize; + let first = self + .stack + .len() + .checked_sub(len) + .ok_or_else(|| malformed("array with too few elements".to_string()))?; + + // The running total belongs to this literal and goes with + // it. `Op::CheckSize` is what filled it in, one element at + // a time, and what raised `ErrorDataTooLarge` against the + // element that tipped it over (`eval/expr.rs:307-330`). + // + // Only if there was one: an empty literal emits no + // `CheckSize` and pushed nothing, so popping here would + // take the *enclosing* literal's total — `[a, [], b]`. + if len > 0 { + self.sizes.pop(); + } + + // Flattened, as rhai does, so a shared cell is copied in + // rather than aliased. + let array: Array = self.stack.drain(first..).map(Dynamic::flatten).collect(); + self.stack.push(Dynamic::from_array(array)); + } + + code::tag::MAKE_MAP => { + let len = small(1)? as usize; + let first = self + .stack + .len() + .checked_sub(2 * len + 1) + .ok_or_else(|| malformed("map with too few operands".to_string()))?; + // As for `MakeArray`: nothing was pushed for a literal + // with no computed entries, so nothing may be popped. + if len > 0 { + self.sizes.pop(); + } + + let mut parts = self.stack.drain(first..); + let template = parts.next().expect("checked above"); + let mut map = template + .try_cast::() + .ok_or_else(|| malformed("map literal without a template".to_string()))?; + while let Some(key) = parts.next() { + let value = parts.next().expect("pairs, checked above"); + let key = key.into_immutable_string().map_err(|actual| { + malformed(format!("map key is a {actual}, not a string")) + })?; + // Flattened as rhai does, so a shared cell is copied + // in rather than aliased. + map.insert(key.as_str().into(), value.flatten()); + } + drop(parts); + self.stack.push(Dynamic::from_map(map)); + } + + code::tag::CHECK_ARRAY_SIZE | code::tag::CHECK_MAP_SIZE => { + let index = small(1)?; + let map = tag == code::tag::CHECK_MAP_SIZE; + self.check_size(index, map, pos())?; + } + + code::tag::SWITCH => { + let index = u32::from(small(1)?); + let table = program + .switch(index) + .ok_or_else(|| malformed(format!("no switch {index}")))?; + let subject = self.pop()?; + // Always a jump: an arm that matched nothing still has the + // default to go to. + transfer!(table.dispatch(&subject) as usize); + continue; + } + + code::tag::LOAD_SHARED => { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + // Cloned, not flattened: cloning a shared `Dynamic` clones + // the `Rc`, which is the capture. + self.stack.push(scope.get_mut_by_index(index).clone()); + } + + // Emitted only for a closure capture, which cannot be parsed + // under `no_closure`. + #[cfg(not(feature = "no_closure"))] + code::tag::SHARE | code::tag::SHARE_NAMED => { + let entry = if tag == code::tag::SHARE { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + Some(index) + } else { + let name_index = u32::from(small(1)?); + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + // The resolver gets first refusal, and a name it + // answers is not shared at all (`eval/stmt.rs:998`). + if self.resolve_var(name, scope, pos())?.is_some() { + pc += width; + continue; + } + // `iter_raw` walks the scope from the top down, which is + // the order shadowing wants — the first match is the + // live one — but it counts from the other end than + // `get_mut_by_index` does, so the position has to be + // turned back round. Rhai reaches the same entry + // through `Scope::search`, which is not public + // (`eval/stmt.rs:1009`). + let depth = scope.len(); + let found = scope + .iter_raw() + .position(|(entry, ..)| entry == name) + .map(|from_top| depth - 1 - from_top); + Some(found.ok_or_else(|| missing(name, pos()))?) + }; + + if let Some(index) = entry { + let value = scope.get_mut_by_index(index); + if !value.is_shared() { + *value = value.take().into_shared(); + } + } + } + + code::tag::MAKE_CLOSURE => { + let index = u32::from(small(1)?); + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + // Unvalidated, because `anon$…` is not a name a script could + // have written and the validating constructors refuse it. + // Nothing unsound rides on that check — a name that will + // not resolve simply fails when the pointer is called. + self.stack.push( + FnPtr { + name: name.into(), + curry: ThinVec::new(), + #[cfg(not(feature = "no_function"))] + env: None, + typ: FnPtrType::Normal, + } + .into(), + ); + } + + #[cfg(not(feature = "no_closure"))] + code::tag::IS_SHARED => { + let value = self.pop()?; + self.stack.push(value.is_shared().into()); + } + + code::tag::MAKE_FN_PTR => { + let name = self.pop()?; + let name = name + .into_immutable_string() + .map_err(|actual| self.mismatch::(actual, pos()))?; + // Validates that the name is an identifier, as rhai's own + // `Fn(..)` does (`func/call.rs:1215`). + let pointer = FnPtr::new(name).map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos()); + } + err + })?; + self.stack.push(pointer.into()); + } + + code::tag::CURRY => { + let argc = code[pc + 1] as usize; + let at = self + .stack + .len() + .checked_sub(argc + 1) + .ok_or_else(|| malformed("curry is missing its target".into()))?; + let mut pointer = self.stack[at] + .clone() + .try_cast::() + .ok_or_else(|| self.mismatch::(self.stack[at].type_name(), pos()))?; + for value in self.stack.drain(at + 1..) { + pointer.add_curry(value); + } + self.stack.truncate(at); + self.stack.push(pointer.into()); + } + + code::tag::CALL_FN_PTR + | code::tag::CALL_FN_PTR_METHOD + | code::tag::CALL_FN_PTR_ON_LOCAL + | code::tag::CALL_FN_PTR_ON_NAMED + | code::tag::CALL_FN_PTR_ON_THIS => { + let argc = code[pc + 1] as usize; + let method = tag != code::tag::CALL_FN_PTR; + let receiver = match tag { + code::tag::CALL_FN_PTR_ON_LOCAL => Some(Receiver::Local(small(2)?)), + code::tag::CALL_FN_PTR_ON_NAMED => { + Some(Receiver::Named(u32::from(small(2)?))) + } + code::tag::CALL_FN_PTR_ON_THIS => Some(Receiver::This), + _ => None, + }; + let value = + self.call_fn_ptr(program, argc, method, receiver, scope, base, pos())?; + self.stack.push(value); + } + + code::tag::INTERPOLATE_START => { + self.stack.push(self.engine.const_empty_string().into()); + } + + code::tag::INTERPOLATE_APPEND => { + let segment = self.pop()?; + self.append_segment(segment, pos())?; + } + + code::tag::INTERPOLATE_END => { + let buffer = self.pop()?; + let text = buffer + .into_immutable_string() + .map_err(|_| malformed("interpolation lost its buffer".into()))?; + // Interned, as rhai does: the same rendered string in ten + // places is one allocation, which is the whole reason the + // engine keeps an interner. + let value = self.engine.get_interned_string(text.as_str()); + self.stack.push(value.into()); + } + + code::tag::CHAIN => { + let index = u32::from(small(1)?); + let chain = program + .chain(index) + .ok_or_else(|| malformed(format!("no chain {index}")))?; + let value = self.run_chain(program, chain, scope, base, pos())?; + self.stack.push(value); + } + + code::tag::UNWIND_TO => { + let depth = small(1)?; + let target = base + depth as usize; + if target > scope.len() { + return Err(malformed(format!( + "unwind to {target} past a scope of {}", + scope.len() + ))); + } + scope.rewind(target); + } + + code::tag::TICK => self.engine.track_operation(&mut self.global, pos())?, + + code::tag::CHECKPOINT => self.unwind_floor = scope.len(), + + code::tag::PUSH_HANDLER | code::tag::PUSH_HANDLER_VAR => { + let target = wide(1)? as usize; + let catch_var = if tag == code::tag::PUSH_HANDLER_VAR { + Some(u32::from(small(5)?)) + } else { + None + }; + self.handlers.push(Handler { + target, + catch_var, + operands: self.stack.len(), + scope_len: scope.len(), + iters: self.iterators.len(), + caught: None, + }); + } + + code::tag::POP_HANDLER => { + self.handlers.pop(); + } + + code::tag::ITER_INIT => { + let iterable = self.pop()?; + self.iter_init(iterable, pos())?; + } + + code::tag::ITER_DROP => { + self.iterators.pop(); + } + + code::tag::ITER_NEXT | code::tag::ITER_NEXT_INDEXED => { + let exit = wide(1)? as usize; + let iteration = self + .iterators + .last_mut() + .ok_or_else(|| malformed("no iterator to advance".to_string()))?; + + let Some(item) = iteration.items.next() else { + self.iterators.pop(); + transfer!(exit); + continue; + }; + + // Counted before the item is unwrapped, as rhai does, so a + // loop long enough to wrap the counter is an error rather + // than a wrap. + iteration.count = iteration.count.checked_add(1).ok_or_else(|| { + Box::new(EvalAltResult::ErrorArithmetic( + format!("for-loop counter overflow: {}", iteration.count), + pos(), + )) + })?; + let count = iteration.count; + + // A fallible iterator's error is positioned at the + // iterable, and only if it brought none of its own + // (`eval/stmt.rs:749`). + let value = item.map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos()); + } + err + })?; + + if tag == code::tag::ITER_NEXT_INDEXED { + self.stack.push(Dynamic::from(count)); + } + self.stack.push(value.flatten()); + } + + code::tag::STORE_SHARED => { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + let value = self.pop()?; + // Through the cell: a closure made in an earlier iteration + // shares this slot, and rhai writes into it rather than + // replacing it (`eval/stmt.rs:752`). + *place(scope.get_mut_by_index(index), "", pos())? = value; + } + + code::tag::THROW => { + // Flattened, as rhai does, so a shared cell is thrown as + // its value rather than as the cell. + let value = self.pop()?.flatten(); + return Err(Box::new(EvalAltResult::ErrorRuntime(value, pos()))); + } + + code::tag::RETURN => { + let value = self.stack.pop().unwrap_or(Dynamic::UNIT); + // Whatever else this frame left behind goes with it, so a + // caller's stack is exactly as it was. + self.stack.truncate(stack_base); + return Ok(value); + } + + // `code::width` already refused anything it does not know, so + // this is unreachable — but a wildcard is what stops a new tag + // from silently falling through to the next instruction. + _ => return Err(malformed(format!("unknown instruction {tag:#04x} at {pc}"))), + } + + pc += width; + } + } +} + +/// The `this` register, reached by hand-built chunks. +/// +/// The compiler does not emit any of these yet — it still refuses a body that +/// mentions `this` — so this is the only thing that executes them until it does. +/// Worth having on its own account regardless: what a hand-made artifact can say +/// is exactly what a verifier-plus-VM has to survive. +#[cfg(test)] +mod tests { + use super::*; + use crate::grain::bytecode::{assemble, Chain, Chunk, Op, Positions, Step, Strings, Tail}; + use crate::grain::program::{Function, Parts}; + use crate::{CallFnOptions, Engine, Scope, INT}; + + /// A program of nullary functions, named `f` upwards in the order given. + /// + /// The main chunk does nothing: everything here is entered through + /// [`Vm::call_fn_with_options`], which is the only thing that can bind a + /// receiver. + fn program_of(bodies: &[&[Op]], consts: Vec) -> Program<'static> { + program_with(bodies, consts, Vec::new(), Vec::new()) + } + + /// The same, for the chain instruction, whose record lives in a pool rather + /// than in the code. + fn program_with_chains( + bodies: &[&[Op]], + consts: Vec, + chains: Vec, + ) -> Program<'static> { + program_with(bodies, consts, chains, Vec::new()) + } + + /// The general form. Name indices are positions in `NAMES`. + fn program_with( + bodies: &[&[Op]], + consts: Vec, + chains: Vec, + residuals: Vec, + ) -> Program<'static> { + /// `f` and `g` are the functions; the rest are for chain steps to name. + const NAMES: [&str; 4] = ["f", "g", "push", "len"]; + + let mut all = vec![Op::Unit, Op::Return]; + let mut spans = Vec::new(); + for body in bodies { + let start = all.len(); + all.extend_from_slice(body); + spans.push(start..all.len()); + } + + // Assembling a prefix gives the byte offset that prefix ends at, which + // is what a chunk is measured in. + let end_of = |ops: usize| { + assemble(&all[..ops]) + .expect("the test ops must assemble") + .0 + .len() as u32 + }; + let (code, _) = assemble(&all).expect("the test ops must assemble"); + + let functions = spans + .iter() + .enumerate() + .map(|(index, span)| Function { + name: index as u32, + params: Vec::new(), + this_type: None, + takes_this: false, + chunk: Chunk::new(end_of(span.start), end_of(span.end), 8), + }) + .collect(); + + Program::new( + code.into(), + Chunk::new(0, end_of(2), 8), + functions, + Parts { + positions: Positions::default(), + residuals, + consts, + names: Strings::new(NAMES), + tokens: Vec::new(), + assign_ops: Vec::new(), + chains, + switches: Vec::new(), + lib: None, + #[cfg(not(feature = "no_module"))] + resolver: None, + source: None, + }, + ) + } + + /// The one function `f`, for the cases that need no callee. + fn one(ops: &[Op], consts: Vec) -> Program<'static> { + program_of(&[ops], consts) + } + + fn call(program: &Program, this: Option<&mut Dynamic>) -> Result> { + let engine = Engine::new(); + let mut options = CallFnOptions::new().eval_ast(false); + options.this_ptr = this; + Vm::new(&engine).call_fn_with_options(options, &mut Scope::new(), program, "f", ()) + } + + #[test] + fn a_bound_receiver_is_what_load_this_pushes() { + let program = one(&[Op::LoadThis, Op::Return], Vec::new()); + let mut this = Dynamic::from(7 as INT); + assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); + } + + #[test] + fn reading_an_unbound_receiver_is_an_error() { + let program = one(&[Op::LoadThis, Op::Return], Vec::new()); + let err = *call(&program, None).unwrap_err(); + assert!( + matches!( + &err, + EvalAltResult::ErrorInFunctionCall(_, _, inner, _) + if matches!(**inner, EvalAltResult::ErrorUnboundThis(..)) + ), + "expected an unbound `this`, got {err:?}" + ); + } + + /// `this = v` checks boundness before evaluating `v`, which is the whole + /// reason [`Op::RequireThis`] is a separate instruction. + #[test] + fn assigning_to_an_unbound_receiver_is_caught_before_the_value_runs() { + let program = one(&[Op::RequireThis, Op::Unit, Op::Return], Vec::new()); + let err = *call(&program, None).unwrap_err(); + assert!( + matches!( + &err, + EvalAltResult::ErrorInFunctionCall(_, _, inner, _) + if matches!(**inner, EvalAltResult::ErrorUnboundThis(..)) + ), + "expected an unbound `this`, got {err:?}" + ); + } + + #[test] + fn a_write_through_this_reaches_the_hosts_value() { + let program = one( + &[ + Op::Const(0), + Op::AssignThis { op: None }, + Op::Unit, + Op::Return, + ], + vec![Dynamic::from(9 as INT)], + ); + let mut this = Dynamic::from(1 as INT); + assert!(call(&program, Some(&mut this)).is_ok()); + assert_eq!(this.as_int(), Ok(9)); + } + + /// Rhai reaches `this` through the caller's storage, so a body that mutates + /// and then raises has already written. + #[test] + fn a_write_through_this_survives_a_failure_after_it() { + let program = one( + &[ + Op::Const(0), + Op::AssignThis { op: None }, + Op::Const(1), + Op::Throw, + ], + vec![Dynamic::from(9 as INT), Dynamic::from("boom")], + ); + let mut this = Dynamic::from(1 as INT); + assert!(call(&program, Some(&mut this)).is_err()); + assert_eq!(this.as_int(), Ok(9)); + } + + /// A callee gets `None`, whatever its caller was holding + /// (`func/call.rs:669`). No conditional makes that true — every ordinary + /// call goes through `call_compiled`, which installs `None`. + /// + /// Worth testing at all because the failure is invisible: a register that + /// leaked would only show up in a callee that reads `this`, and reading a + /// value that happens to be there looks like success. + #[test] + fn a_receiver_is_not_inherited_by_a_callee() { + // `f` has a receiver and calls `g`, which reads one it was never given. + let program = program_of( + &[ + &[ + Op::Call { + name: 1, + argc: 0, + op: None, + }, + Op::Return, + ], + &[Op::LoadThis, Op::Return], + ], + Vec::new(), + ); + + let mut this = Dynamic::from(7 as INT); + let err = *call(&program, Some(&mut this)).unwrap_err(); + assert!( + format!("{err:?}").contains("ErrorUnboundThis"), + "expected `g` to have no receiver, got {err:?}" + ); + } + + /// And the caller still has its own afterwards. + #[test] + fn a_callees_frame_does_not_disturb_the_callers_receiver() { + let program = program_of( + &[ + &[ + Op::Call { + name: 1, + argc: 0, + op: None, + }, + Op::Pop, + Op::LoadThis, + Op::Return, + ], + &[Op::Unit, Op::Return], + ], + Vec::new(), + ); + + let mut this = Dynamic::from(7 as INT); + assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); + } + + /// A chain rooted at `this` whose method is a chunk of ours: the receiver + /// becomes the callee's `this`, which is the binding rhai does at + /// `func/call.rs:649-655` and the only place a method call differs from a + /// plain one. + #[test] + fn a_method_step_reaching_a_chunk_binds_the_receiver() { + let program = program_with_chains( + // `f` is `this.g()`; `g` is `this`. + &[&[Op::Chain(0), Op::Return], &[Op::LoadThis, Op::Return]], + Vec::new(), + vec![Chain { + root: Root::This { + pos: Position::NONE, + }, + steps: vec![Step::Method { + name: 1, // `g` + argc: 0, + operand: 0, + pos: Position::NONE, + }], + tail: Tail::Read, + operands: 0, + }], + ); + + let mut this = Dynamic::from(7 as INT); + assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); + } + + /// And a write inside that callee travels back out through both frames: the + /// callee's register, the chain's root write-back, then the host's pointer. + #[test] + fn a_write_inside_a_method_step_reaches_the_host() { + let program = program_with_chains( + // `f` is `this.g()`; `g` is `this = 9`. + &[ + &[Op::Chain(0), Op::Return], + &[ + Op::Const(0), + Op::AssignThis { op: None }, + Op::Unit, + Op::Return, + ], + ], + vec![Dynamic::from(9 as INT)], + vec![Chain { + root: Root::This { + pos: Position::NONE, + }, + steps: vec![Step::Method { + name: 1, // `g` + argc: 0, + operand: 0, + pos: Position::NONE, + }], + tail: Tail::Read, + operands: 0, + }], + ); + + let mut this = Dynamic::from(1 as INT); + assert!(call(&program, Some(&mut this)).is_ok()); + assert_eq!(this.as_int(), Ok(9)); + } + + /// A fragment the compiler could not lower still sees the receiver. Without + /// this the walker would be handed `None` and report `ErrorUnboundThis` for + /// a `this` the surrounding instructions can read perfectly well. + #[test] + fn a_residual_fragment_reads_the_frames_receiver() { + let program = program_with( + &[&[ + Op::EvalAst { + residual: 0, + rewind_scope: false, + }, + Op::Return, + ]], + Vec::new(), + Vec::new(), + vec![Expr::ThisPtr(Position::NONE)], + ); + + let mut this = Dynamic::from(7 as INT); + assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); + } + + /// A chain rooted at `this` mutates the caller's value rather than a copy. + /// That is the whole reason `Root::This` is not `Root::Temporary`. + #[test] + fn a_chain_rooted_at_this_mutates_the_hosts_value() { + let program = program_with_chains( + &[&[Op::Const(0), Op::Chain(0), Op::Return]], + vec![Dynamic::from(2 as INT)], + vec![Chain { + root: Root::This { + pos: Position::NONE, + }, + steps: vec![Step::Method { + name: 2, // `push` + argc: 1, + operand: 0, + pos: Position::NONE, + }], + tail: Tail::Read, + operands: 1, + }], + ); + + let mut this = Dynamic::from(vec![Dynamic::from(1 as INT)]); + assert!(call(&program, Some(&mut this)).is_ok()); + + let array = this.into_array().expect("still an array"); + let items: Vec = array.iter().map(|v| v.as_int().unwrap()).collect(); + assert_eq!(items, vec![1, 2]); + } + + /// `f(this, ..)` is rhai's method-call rewrite, so the receiver goes by + /// reference and a mutating native reaches the caller's value. + #[test] + fn this_as_a_first_argument_goes_by_reference() { + let program = one( + &[ + Op::LoadThis, + Op::Const(0), + Op::CallRef { + name: 2, // `push` + argc: 2, + receiver: Receiver::This, + }, + Op::Return, + ], + vec![Dynamic::from(2 as INT)], + ); + + let mut this = Dynamic::from(vec![Dynamic::from(1 as INT)]); + if let Err(err) = call(&program, Some(&mut this)) { + panic!("expected the push to succeed, got {err:?}"); + } + + let items: Vec = this + .into_array() + .expect("still an array") + .iter() + .map(|v| v.as_int().unwrap()) + .collect(); + assert_eq!(items, vec![1, 2]); + } + + /// The snapshot is pushed before the other arguments, so an unbound + /// receiver is what fails — not whatever the arguments would have done. + #[test] + fn an_unbound_this_beats_a_failing_argument() { + let program = one( + &[ + Op::LoadThis, + Op::LoadNamed(3), // `len`, which is no variable + Op::CallRef { + name: 2, + argc: 2, + receiver: Receiver::This, + }, + Op::Return, + ], + Vec::new(), + ); + + let err = *call(&program, None).unwrap_err(); + assert!( + format!("{err:?}").contains("ErrorUnboundThis"), + "expected the receiver to fail first, got {err:?}" + ); + } + + #[test] + fn a_chain_rooted_at_an_unbound_this_is_an_error() { + let program = program_with_chains( + &[&[Op::Chain(0), Op::Return]], + Vec::new(), + vec![Chain { + root: Root::This { + pos: Position::NONE, + }, + steps: vec![Step::Method { + name: 3, // `len` + argc: 0, + operand: 0, + pos: Position::NONE, + }], + tail: Tail::Read, + operands: 0, + }], + ); + + let err = *call(&program, None).unwrap_err(); + assert!( + format!("{err:?}").contains("ErrorUnboundThis"), + "expected an unbound `this`, got {err:?}" + ); + } + + #[test] + fn a_read_only_receiver_refuses_the_write_and_is_left_alone() { + let program = one( + &[ + Op::Const(0), + Op::AssignThis { op: None }, + Op::Unit, + Op::Return, + ], + vec![Dynamic::from(9 as INT)], + ); + let mut this = Dynamic::from(1 as INT).into_read_only(); + let err = *call(&program, Some(&mut this)).unwrap_err(); + assert!( + matches!( + &err, + EvalAltResult::ErrorInFunctionCall(_, _, inner, _) + // Named for an expression that has no name. + if matches!(&**inner, EvalAltResult::ErrorAssignmentToConstant(name, ..) if name.is_empty()) + ), + "expected a refused write to a constant, got {err:?}" + ); + assert_eq!(this.as_int(), Ok(1)); + } +} diff --git a/src/lib.rs b/src/lib.rs index e40ea3748..0e611246b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,6 +121,8 @@ pub mod config; mod engine; mod eval; mod func; +#[cfg(feature = "grain")] +pub mod grain; mod module; mod optimizer; pub mod packages; diff --git a/src/module/mod.rs b/src/module/mod.rs index a216d4c08..bdb6e06c4 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -373,10 +373,13 @@ impl FuncRegistration { /// /// # Parameter Examples /// - /// `"foo: &str"` <- parameter name = `foo`, type = `&str` - /// `"bar"` <- parameter name = `bar`, type unknown - /// `"_: i64"` <- parameter name unknown, type = `i64` - /// `"MyType"` <- parameter name unknown, type = `MyType` + /// `"foo: &str"` <- parameter name = `foo`, type = `&str` + /// + /// `"bar"` <- parameter name = `bar`, type unknown + /// + /// `"_: i64"` <- parameter name unknown, type = `i64` + /// + /// `"MyType"` <- parameter name unknown, type = `MyType` #[cfg(feature = "metadata")] #[must_use] pub fn with_params_info>(mut self, params: impl IntoIterator) -> Self { diff --git a/tests/fn_ptr.rs b/tests/fn_ptr.rs index 46a7ddda3..94aa00d11 100644 --- a/tests/fn_ptr.rs +++ b/tests/fn_ptr.rs @@ -150,6 +150,30 @@ fn test_fn_ptr_call() { assert_eq!(result, 42); } +/// A bare function name is a function pointer, and stays one after something +/// has set `always_search_scope`. +/// +/// That flag means "do not trust the parse-time variable indices". A name that +/// resolves to a function is not a variable and has no index to distrust, but +/// the check for one used to sit behind the flag — so an `eval` that changed +/// the scope made every later use of a bare function name report it as an +/// unknown variable. +#[test] +#[cfg(not(feature = "no_function"))] +fn test_fn_ptr_from_bare_name_survives_a_scope_change() { + let engine = Engine::new(); + + // Called function-style, which `no_object` leaves in the language where it + // removes the `f.call(..)` spelling. + assert_eq!(engine.eval::("fn dbl(x) { x * 2 } let f = dbl; call(f, 4)").unwrap(), 8); + + // And the same once `eval` has changed the scope. + assert_eq!(engine.eval::(r#"fn dbl(x) { x * 2 } eval("let m = 2;"); let f = dbl; call(f, 4)"#).unwrap(), 8,); + + // A variable of the same name still wins, which is what the flag is for. + assert_eq!(engine.eval::(r#"fn dbl(x) { x * 2 } eval("let m = 2;"); let dbl = 7; dbl"#).unwrap(), 7,); +} + #[test] #[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_object"))] diff --git a/tests/grain/allocation.rs b/tests/grain/allocation.rs new file mode 100644 index 000000000..22c8d6f4e --- /dev/null +++ b/tests/grain/allocation.rs @@ -0,0 +1,318 @@ +//! What the AST costs, and what the VM costs instead. +//! +//! This is the instrument the project's premise rests on. Rhai's tree costs +//! ~24 bytes of device heap per minified source byte, and traffic-light's +//! `firmware/src/script.rs:88-92` records something the retained figure misses: +//! "rhai's parser allocates well past the size of the tree it finally keeps, so +//! the peak during `compile` is what has to fit, not the result." So this +//! tracks peak as well as retained — traffic-light's own harness tracks only +//! retained, and peak is the number that actually bounds script size. +//! +//! # Why this is one test in its own file +//! +//! The counters are process-global, and cargo runs tests on parallel threads. +//! Any concurrent allocation lands in the same counters and corrupts every +//! reading. One `#[test]`, in its own binary, is what keeps the numbers real. +//! traffic-light learned the same thing (commit bee5fb5, "Fold the allocation +//! measurement into the one test that owns the counters"). +//! +//! # What these numbers are not +//! +//! Host figures. They do not transfer to a device by halving: traffic-light +//! measured the engine ratio at 0.71 rather than 0.5, and found the AST does +//! not shrink on 32-bit at all, because `Dynamic`, `i64` and `f32` fields are +//! the same width either way. Device numbers have to come from a device. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicIsize, Ordering}; + +use rhai::grain::Compiler; +use rhai::Engine; + +static LIVE: AtomicIsize = AtomicIsize::new(0); +static COUNT: AtomicIsize = AtomicIsize::new(0); +static PEAK: AtomicIsize = AtomicIsize::new(0); + +/// Live allocations by size class, `SIZE_CLASSES[i-1] < size <= SIZE_CLASSES[i]`. +/// The small classes are what a per-allocation header punishes, and an AST is +/// mostly small classes. +static BUCKETS: [AtomicIsize; 6] = [AtomicIsize::new(0), AtomicIsize::new(0), AtomicIsize::new(0), AtomicIsize::new(0), AtomicIsize::new(0), AtomicIsize::new(0)]; +const SIZE_CLASSES: [usize; 6] = [8, 16, 32, 64, 256, usize::MAX]; + +fn bucket_of(size: usize) -> usize { + SIZE_CLASSES.iter().position(|&c| size <= c).unwrap_or(5) +} + +struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + let now = LIVE.fetch_add(l.size() as isize, Ordering::Relaxed) + l.size() as isize; + PEAK.fetch_max(now, Ordering::Relaxed); + COUNT.fetch_add(1, Ordering::Relaxed); + BUCKETS[bucket_of(l.size())].fetch_add(1, Ordering::Relaxed); + unsafe { System.alloc(l) } + } + + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + LIVE.fetch_sub(l.size() as isize, Ordering::Relaxed); + COUNT.fetch_sub(1, Ordering::Relaxed); + BUCKETS[bucket_of(l.size())].fetch_sub(1, Ordering::Relaxed); + unsafe { System.dealloc(p, l) } + } + + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + let now = LIVE.fetch_add(new as isize - l.size() as isize, Ordering::Relaxed) + new as isize - l.size() as isize; + PEAK.fetch_max(now, Ordering::Relaxed); + BUCKETS[bucket_of(l.size())].fetch_sub(1, Ordering::Relaxed); + BUCKETS[bucket_of(new)].fetch_add(1, Ordering::Relaxed); + unsafe { System.realloc(p, l, new) } + } +} + +#[global_allocator] +static ALLOC: Counting = Counting; + +fn live() -> isize { + LIVE.load(Ordering::Relaxed) +} + +fn count() -> isize { + COUNT.load(Ordering::Relaxed) +} + +fn buckets() -> [isize; 6] { + std::array::from_fn(|i| BUCKETS[i].load(Ordering::Relaxed)) +} + +/// What producing `T` cost, and what it goes on costing while held. +struct Measured { + value: T, + /// Still-live bytes attributable to the returned value. + bytes: isize, + /// Still-live allocations attributable to the returned value. + count: isize, + /// High-water mark during the call. Exceeds `bytes` by whatever the + /// producer allocated and freed along the way — for rhai's parser, a lot. + peak: isize, + /// Still-live allocations by size class. + buckets: [isize; 6], +} + +fn measure(f: impl FnOnce() -> T) -> Measured { + let base_bytes = live(); + let base_count = count(); + let base_buckets = buckets(); + PEAK.store(base_bytes, Ordering::Relaxed); + + let value = f(); + + let peak = PEAK.load(Ordering::Relaxed) - base_bytes; + let after = buckets(); + + Measured { + bytes: live() - base_bytes, + count: count() - base_count, + peak, + buckets: std::array::from_fn(|i| after[i] - base_buckets[i]), + value, + } +} + +/// A synthetic control, shaped like a real script: helper functions, integer +/// and float arithmetic, arrays, and a loop driving them. +/// +/// `FOLLOW` is the real one, and the figures that matter are measured against +/// it. This is here so the per-source-byte and peak/retained ratios have a +/// second shape to be read against. +const SCRIPT: &str = r#" +fn clamp(v, lo, hi) { + if v < lo { lo } else if v > hi { hi } else { v } +} + +fn ease(t) { + let x = clamp(t, 0.0, 1.0); + x * x * (3.0 - 2.0 * x) +} + +fn blend(a, b, t) { + let k = ease(t); + (a * (1.0 - k) + b * k) +} + +let channels = [0, 0, 0, 0, 0, 0, 0, 0]; +let phase = 0.0; +let step = 0.125; + +for frame in 0..64 { + phase += step; + if phase > 1.0 { phase -= 1.0; } + + for i in 0..channels.len { + let target = if i % 2 == 0 { 255.0 } else { 64.0 }; + let level = blend(0.0, target, ease(phase)); + channels[i] = clamp(level.to_int(), 0, 255); + } +} + +channels +"#; + +/// The script the premise is actually about — byte-identical to +/// traffic-light's `scripts/follow.rhai`, the one its 24 bytes/source-byte +/// figure was measured against. +const FOLLOW: &str = include_str!("fixtures/follow.rhai"); + +/// Something that lowers all the way, so it can be written and read back. +/// +/// `follow.rhai` cannot yet: it still fragments on property reads and index +/// assignment, and a program holding a fragment refuses to serialize. Until +/// chained lvalues land, the load figures come from a script shaped like the +/// parts of it that do lower — locals, arithmetic, branches and a loop. +const LOADABLE: &str = r#" +let total = 0; +let phase = 0; +let step = 3; +let i = 0; +while i < 64 { + phase += step; + if phase > 100 { phase -= 100; } + if i % 2 == 0 { total += phase * 2; } else { total -= phase; } + i += 1; +} +total; +"#; + +// `SCRIPT` and `follow.rhai` both use floats, and the second is checked in +// byte-identical to the script the 24-bytes-per-source-byte figure came from — +// rewriting it to suit a build would make the number mean something else. So +// the measurement is a default-build one, and says so. +#[test] +#[cfg(not(feature = "no_float"))] +fn allocation_footprint() { + let engine = Engine::new(); + + let engine_cost = measure(Engine::new); + let ast = measure(|| engine.compile(SCRIPT).expect("script must compile")); + let program = measure(|| Compiler::new().compile(&ast.value)); + + let follow_ast = measure(|| engine.compile(FOLLOW).expect("follow.rhai must compile")); + + let source_bytes = SCRIPT.len() as isize; + let per_source_byte = ast.bytes as f64 / source_bytes as f64; + + println!("\nsource {source_bytes} bytes"); + println!("\n{:<24} {:>10} {:>10} {:>10}", "", "bytes", "allocs", "peak"); + println!("{:<24} {:>10} {:>10} {:>10}", "Engine::new", engine_cost.bytes, engine_cost.count, engine_cost.peak); + println!("{:<24} {:>10} {:>10} {:>10}", "engine.compile (AST)", ast.bytes, ast.count, ast.peak); + println!("{:<24} {:>10} {:>10} {:>10}", "Compiler::compile", program.bytes, program.count, program.peak); + + // Not comparable to traffic-light's 24.0: that is a 32-bit device figure + // measured against minified source, this is a host figure against + // unminified source. Comparing them needs follow.rhai measured minified, + // the way the server actually ships it. + println!("\nAST bytes per source byte {per_source_byte:.1} (host, unminified)"); + println!("AST parser peak / retained {:.2}x (the peak is what has to fit)", ast.peak as f64 / ast.bytes as f64); + + println!("\nAST live allocations by size class"); + let labels = ["<=8", "<=16", "<=32", "<=64", "<=256", ">256"]; + for (label, n) in labels.iter().zip(ast.buckets) { + println!(" {label:>6} {n:>8}"); + } + + // What the real script's tree costs, to set against what + // tests/grain/projection.rs says a lowering of it would weigh. + // + // Larger than traffic-light's own 77968 for the same script because this + // builds rhai with default features. Theirs sets `no_module`, which drops + // a `Namespace` (an inline `StaticVec` plus a hash) from every + // `Expr::Variable` payload — and this script has 457 of them. A restricted + // build is what 77968 should be compared against, not this one. + println!("\nfollow.rhai — {} source bytes", FOLLOW.len()); + println!("{:<24} {:>10} {:>10} {:>10}", "engine.compile (AST)", follow_ast.bytes, follow_ast.count, follow_ast.peak); + println!("{:<24} {:>10.1}", "bytes per source byte", follow_ast.bytes as f64 / FOLLOW.len() as f64); + println!("{:<24} {:>10.2}x", "parser peak / retained", follow_ast.peak as f64 / follow_ast.bytes as f64); + println!("\nfollow.rhai AST live allocations by size class"); + for (label, n) in labels.iter().zip(follow_ast.buckets) { + println!(" {label:>6} {n:>8}"); + } + + // What loading an artifact costs, against parsing the same script. + // + // This is the claim the byte encoding exists for. `Program::read` borrows + // its instructions from the buffer, so nothing it retains is proportional + // to how long the script is — only to the distinct names and constants it + // mentions. Parsing retains a node per node. + // The measurement the whole project was for: the real script, the real + // tree, and the artifact that replaces it. + let follow_program = Compiler::new().compile(&follow_ast.value); + assert_eq!(follow_program.residual_count(), 0, "follow.rhai must lower completely, or there is nothing to write",); + let (follow_artifact, follow_table) = follow_program.write_stripped().expect("follow.rhai must be writable"); + let follow_loaded = measure(|| rhai::grain::Program::read(&follow_artifact).expect("must load")); + + println!( + "\nfollow.rhai: {} source bytes\n tree {:>8} bytes retained, {:>5} allocs, {:>8} peak\n \ + artifact {:>8} bytes on the wire ({} of debug table kept off it)\n \ + loaded {:>8} bytes retained, {:>5} allocs", + FOLLOW.len(), + follow_ast.bytes, + follow_ast.count, + follow_ast.peak, + follow_artifact.len(), + follow_table.len(), + follow_loaded.bytes, + follow_loaded.count, + ); + println!(" {:.1}x less retained, and no parser peak at all", follow_ast.bytes as f64 / follow_loaded.bytes as f64,); + + // Measured at several lengths, because the ratio at one length says almost + // nothing. Repeating the same statements grows the instruction stream while + // the set of distinct names stays put — which is exactly the shape the + // claim is about, and the shape a real script has as it gets longer. + println!("\nloading an artifact against parsing the same script"); + println!("{:>7} {:>9} {:>9} {:>9} {:>8} {:>7}", "source", "artifact", "tree", "loaded", "allocs", "ratio"); + + let mut ratios = Vec::new(); + for repeats in [1usize, 4, 16] { + let source = LOADABLE.repeat(repeats); + + let tree = measure(|| engine.compile(&source).expect("must compile")); + let program = Compiler::new().compile(&tree.value); + assert_eq!(program.residual_count(), 0, "LOADABLE must lower completely, or this measures the walker",); + + let (stripped, _) = program.write_stripped().expect("must be writable"); + let loaded = measure(|| rhai::grain::Program::read(&stripped).expect("must load")); + + // Borrowed, not copied: the code section contributes nothing at all to + // what a load retains, which is the whole reason for the byte encoding. + let code = loaded.value.code(); + assert!( + code.as_ptr() >= stripped.as_ptr() && code.as_ptr() as usize <= stripped.as_ptr() as usize + stripped.len(), + "the loaded chunk must point into the artifact, not into a copy of it", + ); + + let ratio = tree.bytes as f64 / loaded.bytes as f64; + ratios.push(ratio); + println!("{:>7} {:>9} {:>9} {:>9} {:>8} {:>6.1}x", source.len(), stripped.len(), tree.bytes, loaded.bytes, loaded.count, ratio,); + } + + // The property, rather than a number that would need updating: what a tree + // retains grows with the program, what a load retains does not. A ratio + // that stopped climbing would mean something in the loader had started + // scaling with length. + assert!(ratios[2] > ratios[1] && ratios[1] > ratios[0], "the saving must grow with the program, got {ratios:?}",); + + // The instrument has to be working before anything can lean on it. + assert!(ast.count > 0 && ast.bytes > 0, "the counters saw nothing; the global allocator is not installed",); + assert!(ast.peak >= ast.bytes, "peak ({}) below retained ({}) means peak tracking is broken", ast.peak, ast.bytes,); + + // A lowered program retains its pools rather than a node per node, so it + // costs a fraction of the tree it came from. A `Program` that cost about + // what the tree cost would mean the script fell back to fragments, which + // hold real `Expr` trees — the one case where compiling saves nothing. + assert!(program.bytes > 0 && program.bytes < ast.bytes, "a lowered program must retain less than its tree, got {} against {}", program.bytes, ast.bytes,); + + drop(program); + drop(ast); + drop(engine_cost); +} diff --git a/tests/grain/call_fn.rs b/tests/grain/call_fn.rs new file mode 100644 index 000000000..b0d835fe1 --- /dev/null +++ b/tests/grain/call_fn.rs @@ -0,0 +1,186 @@ +//! Calling one compiled function, with the environment and receiver rhai gives +//! it. +//! +//! A program's library, source and module resolver used to be installed around +//! its *main chunk* only, so a function reached through `call_fn` ran without +//! them. That is not a corner: the compiler leaves anything it cannot lower as +//! an AST in the library, and rhai finds it only in `global.lib`. +//! +//! The other half is `this`. An event handler bound to its state through +//! `bind_this_ptr` is the common shape of a `call_fn` caller, and the whole of +//! what these check is that a `Vm` answers such a call the way an `Engine` +//! does — including that a write through `this` lands in the caller's value. + +use rhai::grain::format::WriteError; +use rhai::grain::{Compiler, Vm}; +use rhai::{CallFnOptions, Dynamic, Engine, EvalAltResult, Scope, INT}; + +/// A receiver for the methods below. +fn holder(count: INT) -> Dynamic { + Dynamic::from_map([("count".into(), Dynamic::from(count))].into_iter().collect()) +} + +/// A bare `eval` statement is what keeps the callee an AST in the program's +/// library — it can declare into the caller's scope, which no slot model can +/// account for — and the library is the only thing that can answer the method +/// call below. In *expression* position it becomes a fragment instead, which +/// leaves the function compiled and would not exercise this at all. +/// +/// It used to be `this` here. That stopped being unlowerable, which would have +/// left this test passing while checking nothing. +const PROGRAM: &str = r#" + fn labelled() { eval("1"); 42 } + fn outer(m) { m.labelled() } +"#; + +/// The same shape, but the un-lowered function's name is also a rhai built-in +/// (`Dynamic::tag`). Without the library the call does not fail — it silently +/// resolves to the built-in getter and answers 0. +const SHADOWED: &str = r#" + fn tag() { eval("1"); 42 } + fn outer(m) { m.tag() } +"#; + +fn walked(engine: &Engine, source: &str) -> INT { + let ast = engine.compile(source).unwrap(); + engine.call_fn(&mut Scope::new(), &ast, "outer", (holder(7),)).unwrap() +} + +fn run(engine: &Engine, source: &str) -> INT { + let ast = engine.compile(source).unwrap(); + let program = Compiler::new().compile(&ast); + // The premise: the callee really is one the compiler left behind, so the + // library has to be installed for the call to resolve at all. Refusing to + // write for *that* reason is the public way to see it. + assert!(matches!(program.write(), Err(WriteError::HasScriptFunctions)), "{source:?} no longer keeps a library, so this would prove nothing (write said {:?})", program.write(),); + Vm::new(engine).call_fn(&mut Scope::new(), &program, "outer", (holder(7),)).unwrap() +} + +#[test] +fn a_call_reaches_a_function_the_compiler_left_to_rhai() { + let engine = Engine::new(); + assert_eq!(walked(&engine, PROGRAM), 42); + assert_eq!(run(&engine, PROGRAM), walked(&engine, PROGRAM)); +} + +#[test] +fn a_missing_library_cannot_be_answered_by_a_builtin_of_the_same_name() { + let engine = Engine::new(); + assert_eq!(walked(&engine, SHADOWED), 42); + assert_eq!(run(&engine, SHADOWED), walked(&engine, SHADOWED)); +} + +/// The environment is now installed once around both halves of the call, so the +/// half that used to have it must not have lost it. +#[test] +fn the_main_chunk_still_runs_before_the_call() { + let engine = Engine::new(); + let ast = engine.compile("fn outer() { 1 } let started = 9;").unwrap(); + let program = Compiler::new().compile(&ast); + + let mut scope = Scope::new(); + let value: INT = Vm::new(&engine) + .call_fn_with_options(CallFnOptions::new().rewind_scope(false), &mut scope, &program, "outer", ()) + .unwrap(); + + assert_eq!(value, 1); + // What the main chunk declared is left in the caller's scope, as + // `Engine::eval_ast_with_scope` would leave it. + assert_eq!(scope.get_value::("started"), Some(9)); +} + +#[test] +fn eval_ast_off_skips_the_main_chunk() { + let engine = Engine::new(); + let ast = engine.compile("fn outer() { 1 } let started = 9;").unwrap(); + let program = Compiler::new().compile(&ast); + + let mut scope = Scope::new(); + let _: INT = Vm::new(&engine) + .call_fn_with_options(CallFnOptions::new().eval_ast(false).rewind_scope(false), &mut scope, &program, "outer", ()) + .unwrap(); + + assert!(scope.get_value::("started").is_none()); +} + +#[test] +fn an_error_inside_a_call_carries_the_program_source() { + let engine = Engine::new(); + let mut ast = engine.compile(r#"fn outer() { throw "boom" }"#).unwrap(); + ast.set_source("handlers.rhai"); + let program = Compiler::new().compile(&ast); + + let err = *Vm::new(&engine).call_fn::(&mut Scope::new(), &program, "outer", ()).unwrap_err(); + + match err { + EvalAltResult::ErrorInFunctionCall(name, source, ..) => { + assert_eq!(name, "outer"); + assert_eq!(source, "handlers.rhai"); + } + other => panic!("expected a wrapped call error, got {other:?}"), + } +} + +/// Both sides of one `call_fn` with a bound receiver: the value it answers, and +/// the caller's `Dynamic` afterwards. +fn bound(engine: &Engine, source: &str, name: &str, state: &mut Dynamic) -> Result { + let ast = engine.compile(source).unwrap(); + let program = Compiler::new().compile(&ast); + + Vm::new(engine) + .call_fn_with_options(CallFnOptions::new().bind_this_ptr(state), &mut Scope::new(), &program, name, (1 as INT,)) + .map_err(|err| format!("{err:?}")) +} + +/// The same through the walker, for the comparison to mean anything. +fn walked_bound(engine: &Engine, source: &str, name: &str, state: &mut Dynamic) -> Result { + let ast = engine.compile(source).unwrap(); + engine + .call_fn_with_options(CallFnOptions::new().bind_this_ptr(state), &mut Scope::new(), &ast, name, (1 as INT,)) + .map_err(|err| format!("{err:?}")) +} + +/// The shape this work exists for: a handler with its state on `this`. +#[test] +fn a_handler_bound_to_its_state_answers_as_the_walker_does() { + let engine = Engine::new(); + const HANDLER: &str = "fn bump(n) { this.count += n; this.count }"; + + let mut state = holder(1); + let mut walked_state = holder(1); + + assert_eq!(bound(&engine, HANDLER, "bump", &mut state), Ok(2)); + assert_eq!(walked_bound(&engine, HANDLER, "bump", &mut walked_state), Ok(2)); + // And the write reached the caller's own `Dynamic`, on both sides. + assert_eq!(format!("{state:?}"), format!("{walked_state:?}")); + assert_eq!(format!("{state:?}"), r#"#{"count": 2}"#); +} + +/// Rhai reaches `this` through the caller's storage, so a handler that writes +/// and then fails has already written. +#[test] +fn a_write_before_a_failure_reaches_the_caller() { + let engine = Engine::new(); + const HANDLER: &str = r#"fn bump(n) { this.count += n; throw "boom" }"#; + + let mut state = holder(1); + let mut walked_state = holder(1); + + assert!(bound(&engine, HANDLER, "bump", &mut state).is_err()); + assert!(walked_bound(&engine, HANDLER, "bump", &mut walked_state).is_err()); + + assert_eq!(format!("{state:?}"), format!("{walked_state:?}")); + assert_eq!(format!("{state:?}"), r#"#{"count": 2}"#); +} + +/// A handler is a chunk now, so a program full of them is an artifact. +#[test] +fn a_program_of_handlers_is_writable() { + let engine = Engine::new(); + let ast = engine.compile("fn on_open(n) { this.count += n } fn on_close() { this.count = 0 }").unwrap(); + let program = Compiler::new().compile(&ast); + + assert_eq!(program.residual_count(), 0); + assert_eq!(program.functions().len(), 2); + assert!(program.write().is_ok(), "got {:?}", program.write()); +} diff --git a/tests/grain/callback.rs b/tests/grain/callback.rs new file mode 100644 index 000000000..a8b33fa89 --- /dev/null +++ b/tests/grain/callback.rs @@ -0,0 +1,213 @@ +//! Natives that call a compiled function back. +//! +//! `[1, 2, 3].map(|x| x * 2)` is the shape: `map` is rhai's, and the pointer it +//! is handed is resolved by rhai's dispatch rather than by ours. This is where +//! the wrappers that make that resolve are held to the walker's behaviour, and +//! where the places it still diverges are pinned rather than left to be +//! discovered. + +// Only the engine is wanted here; the corpus scripts belong to the harnesses +// that run all of them. +use super::corpus; + +use rhai::grain::{Compiler, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +/// Run a source through the VM with the callback wrappers installed. +fn run(engine: &Engine, source: &str) -> Result { + let ast = engine.compile(source).map_err(|err| format!("{err:?}"))?; + let program = Compiler::new().compile(&ast).into_shared(); + Vm::new(engine) + .eval_with_callbacks(&mut Scope::new(), &program) + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")) +} + +/// The same source through the walker, which is what the answer has to be. +fn walk(engine: &Engine, source: &str) -> Result { + let ast = engine.compile(source).map_err(|err| format!("{err:?}"))?; + engine.eval_ast::(&ast).map(|value| format!("{value:?}")).map_err(|err| format!("{err:?}")) +} + +fn agree(source: &str) { + let engine = corpus::engine(); + assert_eq!(walk(&engine, source), run(&engine, source), "{source}"); +} + +/// The array is bound to a variable in every case here, and that is not +/// incidental: a chain rooted at a literal is still a fragment, and a fragment +/// hands the whole expression back to the walker — which would make every one +/// of these pass without the wrappers existing at all. +fn lowered(source: &str) { + let engine = corpus::engine(); + let ast = engine.compile(source).unwrap(); + let program = Compiler::new().compile(&ast); + assert!(program.residual_count() == 0, "{source} still fragments, so it does not test the callback path",); + assert!(program.makes_fn_pointers(), "{source}"); +} + +#[test] +fn a_native_can_call_a_closure_back() { + lowered("let a = [1, 2, 3]; a.map(|x| x * 2)"); + agree("let a = [1, 2, 3]; a.map(|x| x * 2)"); + agree("let a = [1, 2, 3, 4]; a.filter(|x| x % 2 == 0)"); +} + +#[test] +fn a_native_can_call_a_named_function_back() { + agree("fn double(x) { x * 2 } let a = [1, 2, 3]; a.map(Fn(\"double\"))"); +} + +/// A bare function name is a function pointer, not a variable read. +/// +/// The compiler leaves it to rhai — it is a fragment — and rhai used to refuse +/// it, because a program holding any fragment sets `always_search_scope` and +/// the check for a function of that name sat behind the flag. Every spelling +/// below reported `double` as an unknown variable. +#[test] +fn a_bare_function_name_is_a_pointer() { + agree("fn double(x) { x * 2 } let a = [1, 2, 3]; a.map(double)"); + agree("fn double(x) { x * 2 } let r = 0; { let f = double; r = f.call(4); } r"); + agree("fn double(x) { x * 2 } fn apply(f, v) { f.call(v) } apply(double, 4)"); + // A variable of the same name still wins over the function. + agree("fn double(x) { x * 2 } let double = 7; double"); +} + +/// A capture arrives, and has the right value. +/// +/// Multiplication commutes, so this says nothing about which *position* it +/// arrives in — see `a_capturing_closure_reaches_a_native_with_its_arguments_rotated` +/// for that, which is where the answer is unwelcome. +#[test] +#[cfg(not(feature = "no_closure"))] +fn a_capturing_closure_resolves_and_sees_its_capture() { + lowered("let n = 10; let a = [1, 2, 3]; a.map(|x| x * n)"); + agree("let n = 10; let a = [1, 2, 3]; a.map(|x| x * n)"); +} + +/// A capturing closure handed to a native that binds `this` gets its captured +/// values *after* the element instead of before. +/// +/// Not our arithmetic: it is which shape rhai tries first. A capture is a +/// curried value, and `_call_with_extra_args` (`types/fn_ptr.rs:573`) opens +/// with `[this] ++ curry ++ args` — the one order that is never right. Rhai's +/// own closures escape it because they are `Fn*` pointers with the body +/// attached, which is caught two branches earlier at `:538` and rearranged +/// into `curry ++ [this] ++ args`. +/// +/// Ours are `Fn` pointers resolved by name, so rhai reaches them as *natives* +/// and takes the first shape. The proof that this is rhai's behaviour rather +/// than ours is `stock_rhai_does_the_same_to_its_own_native_pointers` below, +/// which reproduces it with no rhaigrain in the picture at all. +/// +/// So: safe for a closure that captures nothing, and for one whose parameters +/// commute. Wrong, silently, otherwise. Non-commutative on purpose here — +/// `n - x` and `x - n` are the same two values in the other order, and only +/// arithmetic that cares can tell them apart. +#[test] +#[cfg(not(feature = "no_closure"))] +fn a_capturing_closure_reaches_a_native_with_its_arguments_rotated() { + let engine = corpus::engine(); + let source = "let n = 10; let a = [1, 2, 3]; a.map(|x| n - x)"; + + assert_eq!(walk(&engine, source), Ok("[9, 8, 7]".to_string())); + assert_eq!(run(&engine, source), Ok("[-9, -8, -7]".to_string())); +} + +/// The same rotation, with nothing of ours involved. +/// +/// `Fn(s)` on a computed name is the one spelling that gets a name-only +/// pointer out of stock rhai, which is what every pointer we make is. Curry it, +/// point it at a native, and hand it to `map`: rhai passes the element first +/// and the curried value second, while `f.call(1)` on the very same pointer +/// passes them the other way round. +#[test] +fn stock_rhai_does_the_same_to_its_own_native_pointers() { + let mut engine = corpus::engine(); + engine.register_fn("nsub", |a: rhai::INT, b: rhai::INT| a - b); + + let curried = "let s = \"ns\" + \"ub\"; let f = Fn(s).curry(10);"; + assert_eq!(walk(&engine, &format!("{curried} [1, 2, 3].map(f)")), Ok("[-9, -8, -7]".to_string()), "rhai puts the element before the curried value",); + assert_eq!(walk(&engine, &format!("{curried} f.call(1)")), Ok("9".to_string()), "and the curried value first when there is no element",); +} + +#[test] +#[cfg(not(feature = "unchecked"))] +fn a_callback_reaching_a_second_one_still_resolves() { + // The limit is raised because reaching a chunk through rhai's dispatch + // costs more call levels than reaching a script function does, and the + // default in a debug build is 8. `a_callback_costs_more_call_levels` + // measures the difference; this is about resolution, not depth. + let mut engine = corpus::engine(); + engine.set_max_call_levels(64); + let source = "let a = [[1, 2], [3, 4]]; a.map(|row| row.map(|x| x + 1))"; + assert_eq!(walk(&engine, source), run(&engine, source)); +} + +#[test] +#[cfg(not(feature = "no_closure"))] +fn a_closure_called_back_sees_a_write_made_after_it_was_made() { + agree("let n = 1; let f = |x| x * n; n = 10; let a = [1, 2, 3]; a.map(f)"); +} + +#[test] +fn an_error_inside_a_callback_still_arrives() { + let engine = corpus::engine(); + let err = run(&engine, "let a = [1, 2, 3]; a.map(|x| throw x)").unwrap_err(); + assert!(err.contains("ErrorInFunctionCall"), "{err}"); +} + +/// Reaching a chunk through rhai's dispatch spends more of the call budget +/// than reaching a script function does, so a callback nests less deeply before +/// `max_call_levels` stops it. +/// +/// Measured rather than asserted at a number, because the number is a property +/// of rhai's dispatch and would drift. What has to hold is the direction: we +/// are stricter, never laxer. A program that would have run out of budget on +/// the walker must not somehow keep going here. +#[test] +#[cfg(not(feature = "unchecked"))] +fn a_callback_costs_more_call_levels() { + let deepest = |depth: usize, run: &dyn Fn(&Engine, &str) -> Result| { + // `depth` closures nested through `map`, each one a boundary, over an + // array nested to match. + let mut source = format!("let a = {}1{}; a", "[".repeat(depth), "]".repeat(depth)); + for level in 0..depth { + source.push_str(&format!(".map(|v{level}| v{level}")); + } + source.push_str(" + 1"); + source.push_str(&")".repeat(depth)); + (1..=64) + .find(|levels| { + let mut engine = corpus::engine(); + engine.set_max_call_levels(*levels); + run(&engine, &source).is_ok() + }) + .unwrap_or(usize::MAX) + }; + + for depth in 1..=3 { + let walker = deepest(depth, &walk); + let vm = deepest(depth, &run); + println!("{depth} nested callback(s): walker needs {walker}, we need {vm}"); + assert!( + vm >= walker, + "depth {depth}: we ran at {vm} levels where the walker needed {walker}, \ + so a budget the walker respects is not being enforced", + ); + } +} + +#[test] +fn without_the_wrappers_the_pointer_does_not_resolve() { + // The whole reason `eval_with_callbacks` exists. A plain eval leaves rhai + // nowhere to look, and the failure is a lookup failure rather than + // anything worse. + let engine = corpus::engine(); + let source = "let a = [1, 2, 3]; a.map(|x| x * 2)"; + let ast = engine.compile(source).unwrap(); + let program = Compiler::new().compile(&ast); + + let err = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).unwrap_err(); + assert!(format!("{err:?}").contains("ErrorFunctionNotFound"), "{err}"); +} diff --git a/tests/grain/corpus/generate.rs b/tests/grain/corpus/generate.rs new file mode 100644 index 000000000..02d84b307 --- /dev/null +++ b/tests/grain/corpus/generate.rs @@ -0,0 +1,534 @@ +// Random rhai scripts, for comparing the VM against the walker on programs +// nobody wrote. +// +// The hand-written corpus proves the constructs someone thought to test. This +// covers the combinations they did not: a `try` inside a `for` inside a +// `switch` arm whose subject is an indexed chain, and the several thousand +// other shapes of that kind. Every one is run both ways and the results +// compared, so a divergence names a script that reproduces it. +// +// Two constraints follow from `fuzz/fuzz_targets/generated.rs` pulling this +// file in with `include!` rather than depending on it: nothing may be imported +// beyond `std`, and the header is `//` rather than `//!`, because an included +// file cannot open with an inner attribute. +// +// What is deliberately not generated: +// +// - Recursion. A function may only call one defined before it, so no script +// can recurse. Running out of call levels is a limit both sides enforce +// but not in lockstep, and it would drown real findings. +// - A closure handed to a native. `a.map(|x| ..)` has a known divergence in +// argument order for captures, pinned in `tests/callback.rs`. Closures are +// generated and called directly instead. +// - An unbounded loop. Every `while` counts a guard variable the body cannot +// name, so it always terminates. + +/// A source of choices: xorshift64*, optionally front-loaded with bytes +/// somebody else picked. +/// +/// The bytes are for `cargo fuzz`. Every grammar decision takes one, so +/// flipping byte *k* of an input changes decision *k* and leaves the rest of +/// the script alone — which is the locality a coverage-guided fuzzer needs to +/// make progress. From a plain seed, or once the bytes run out, it is an +/// ordinary reproducible PRNG. +pub struct Rng { + state: u64, + choices: Vec, + at: usize, +} + +impl Rng { + #[must_use] + pub fn new(seed: u64) -> Self { + Self { + // A zero state is a fixed point for xorshift, and zero is exactly + // the seed somebody reaches for first. + state: if seed == 0 { 0x9e37_79b9_7f4a_7c15 } else { seed }, + choices: Vec::new(), + at: 0, + } + } + + /// Draws from `choices` until they are used up, then from the PRNG. + /// + /// Running out is not a failure and not a truncation: a script that stops + /// being fuzzer-directed part way through is still a valid script, and + /// still worth comparing. + #[must_use] + pub fn from_bytes(choices: &[u8]) -> Self { + let mut seed = 0u64; + for byte in choices.iter().take(8) { + seed = (seed << 8) | u64::from(*byte); + } + Self { choices: choices.to_vec(), at: 0, ..Self::new(seed) } + } + + pub fn next(&mut self) -> u64 { + let mut x = self.state; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.state = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + pub fn below(&mut self, bound: usize) -> usize { + if bound == 0 { + return 0; + } + if let Some(byte) = self.choices.get(self.at) { + self.at += 1; + return usize::from(*byte) % bound; + } + (self.next() % bound as u64) as usize + } + + fn one_of<'a, T>(&mut self, choices: &'a [T]) -> &'a T { + &choices[self.below(choices.len())] + } + + fn chance(&mut self, n: usize) -> bool { + self.below(n) == 0 + } +} + +/// How deep an expression may nest. +/// +/// Rhai's parser has its own limit and rejects anything past it. Staying well +/// under means a rejected parse is a bug in this generator rather than the +/// expected outcome for half the corpus. +const MAX_DEPTH: usize = 3; + +/// How deep blocks and control flow may nest, for the same reason. +/// +/// Rhai counts statements towards the same budget and allows a function body +/// half of what it allows the top level, so this is what keeps a script inside +/// both. Scripts are wide rather than deep as a result, which is the right +/// trade: a `try` inside a `for` finds things, a `try` inside a `for` inside a +/// `while` inside an `if` mostly finds the parser. +const MAX_NESTING: usize = 2; + +/// Operators whose *result* is a boolean whatever their operands turn out to +/// be, so the parser accepts one as a condition. +/// +/// It type-checks that much statically: `if 5 { }` is a parse error, not a +/// runtime one. Comparisons are safe to build out of anything because only the +/// operands can be wrong, and that is a runtime disagreement worth generating. +const COMPARISONS: &[&str] = &["==", "!=", "<", "<=", ">", ">="]; + +/// Operators that are defined for at least one pair of generated types. A pair +/// they are not defined for is a runtime error both sides have to agree on, +/// which is worth generating rather than worth avoiding. +/// +/// `&&` and `||` are not here — they demand boolean operands at parse time, so +/// they belong to [`Generator::condition`] and nowhere else. +const BINARY: &[&str] = &["+", "-", "*", "/", "%", "==", "!=", "<", "<=", ">", ">=", "&", "|"]; + +const METHODS: &[&str] = &["len", "to_string", "abs", "is_empty", "floor", "to_upper"]; + +/// Natives called in *function-call* style, with their arity. +/// +/// Rhai rewrites `f(x, ..)` into `x.f(..)` when the first argument is a plain +/// variable, so `push(a, 2)` mutates `a` and `push([1], 2)` mutates nothing — +/// the same syntax, and only the receiver decides. Half of these mutate and +/// half do not, because whether the write lands is the question. +/// +/// Nothing here can grow a value without bound: an argument reaching one of +/// these is an arbitrary generated integer, and `pad` would turn that into an +/// allocation rather than into a divergence. +const NATIVES: &[(&str, usize)] = &[("push", 2), ("insert", 3), ("remove", 2), ("truncate", 2), ("reverse", 1), ("clear", 1), ("pop", 1), ("shift", 1), ("len", 1), ("is_empty", 1), ("abs", 1), ("to_upper", 1)]; + +pub struct Generator { + rng: Rng, + /// Variables that can be named, innermost last. + vars: Vec, + /// Functions defined so far, with their arity. Only these can be called, + /// and only from after their definition, which is what rules out recursion. + functions: Vec<(String, usize)>, + /// Loop nesting, so `break` and `continue` appear only where they parse. + loops: usize, + depth: usize, + /// Nesting of blocks and control flow, capped for the same reason `depth` + /// is: rhai counts statements towards the same complexity limit, so a + /// `while` inside an `if` inside a `for` runs out of budget on its own. + nesting: usize, + /// Set while generating the inside of an interpolated string, where + /// another one cannot go — the lexer ends the outer string on the inner + /// one's opening backtick. + interpolating: bool, + /// Supplies names that cannot collide with each other. + counter: usize, +} + +impl Generator { + #[must_use] + pub fn new(seed: u64) -> Self { + Self { + rng: Rng::new(seed), + vars: Vec::new(), + functions: Vec::new(), + loops: 0, + depth: 0, + nesting: 0, + interpolating: false, + counter: 0, + } + } + + /// A generator whose choices are made by somebody else's bytes, for + /// `cargo fuzz`. See [`Rng::from_bytes`]. + #[must_use] + pub fn from_bytes(choices: &[u8]) -> Self { + Self { rng: Rng::from_bytes(choices), ..Self::new(0) } + } + + fn name(&mut self, prefix: &str) -> String { + self.counter += 1; + format!("{prefix}{}", self.counter) + } + + /// One whole script. + pub fn script(&mut self) -> String { + let mut out = String::new(); + + for _ in 0..self.rng.below(3) { + out.push_str(&self.function()); + out.push(' '); + } + + // A scope of its own per script, so nothing leaks between them. + self.vars.clear(); + for _ in 0..1 + self.rng.below(5) { + out.push_str(&self.statement()); + out.push(' '); + } + + // Ending on an expression gives the script a value, so a divergence in + // what it computed shows up and not only a divergence in what it left + // behind. + out.push_str(&self.expression()); + out + } + + fn function(&mut self) -> String { + let name = self.name("f"); + let arity = self.rng.below(3); + let params: Vec = (0..arity).map(|i| format!("p{i}")).collect(); + + // A body sees its parameters and nothing else, which is also what rhai + // gives it. + let outer = std::mem::replace(&mut self.vars, params.clone()); + let outer_loops = std::mem::replace(&mut self.loops, 0); + // Rhai allows a function body half the expression depth it allows the + // top level (`MAX_FUNCTION_EXPR_DEPTH`), so a body starts one level in. + let outer_nesting = std::mem::replace(&mut self.nesting, 1); + let mut body = String::new(); + for _ in 0..1 + self.rng.below(3) { + body.push_str(&self.statement()); + body.push(' '); + } + body.push_str(&self.expression()); + self.vars = outer; + self.loops = outer_loops; + self.nesting = outer_nesting; + + // Registered after the body is generated, so the body cannot call it. + self.functions.push((name.clone(), arity)); + format!("fn {name}({}) {{ {body} }}", params.join(", ")) + } + + fn statement(&mut self) -> String { + // Weighted by hand: `let` more often than anything else, because a + // script with no variables cannot exercise slots, chains or capture. + // Anything that nests is dropped from the choices once deep enough, + // which is why the arms are ordered with those last. + let nesting_kinds = if self.nesting < MAX_NESTING { 5 } else { 0 }; + let loop_kinds = if self.loops > 0 { 2 } else { 0 }; + match self.rng.below(5 + nesting_kinds + loop_kinds) { + 0..=2 => { + let value = self.expression(); + let name = self.name("v"); + self.vars.push(name.clone()); + format!("let {name} = {value};") + } + 3 => self.assignment(), + 4 => format!("{};", self.expression()), + + // Nesting, reachable only while there is budget for it. + 5 => { + let cond = self.condition(); + self.nesting += 1; + let then = self.block(); + let otherwise = self.rng.chance(2).then(|| self.block()); + self.nesting -= 1; + match otherwise { + Some(otherwise) => format!("if {cond} {then} else {otherwise}"), + None => format!("if {cond} {then}"), + } + } + 6 => self.nested(Self::bounded_while), + 7 => self.nested(Self::for_loop), + 8 => self.nested(Self::switch), + 9 => self.nested(Self::try_catch), + + // Only reachable inside a loop, per `loop_kinds` above. + 10 => "break;".to_string(), + _ => "continue;".to_string(), + } + } + + fn nested(&mut self, build: fn(&mut Self) -> String) -> String { + self.nesting += 1; + let out = build(self); + self.nesting -= 1; + out + } + + /// A run of statements, with the scope they declared popped afterwards + /// exactly as the VM pops it. + /// + /// Returned without braces so a caller can append to it. Appending to a + /// finished block by trimming its `}` back off is how a body that happens + /// to end in one gets silently truncated. + fn statements(&mut self) -> String { + let mark = self.vars.len(); + let mut out = String::new(); + for _ in 0..1 + self.rng.below(2) { + out.push_str(&self.statement()); + out.push(' '); + } + self.vars.truncate(mark); + out + } + + fn block(&mut self) -> String { + format!("{{ {} }}", self.statements()) + } + + fn assignment(&mut self) -> String { + let Some(target) = self.variable() else { + return format!("{};", self.expression()); + }; + let op = self.rng.one_of(&["=", "+=", "-=", "*="]).to_string(); + format!("{target} {op} {};", self.expression()) + } + + /// `while`, with a guard the body cannot reach. + /// + /// The guard is not pushed onto `vars`, so nothing generated inside can + /// assign to it and the loop always ends. + fn bounded_while(&mut self) -> String { + let guard = self.name("g"); + let limit = 1 + self.rng.below(4); + self.loops += 1; + let body = self.statements(); + self.loops -= 1; + format!("let {guard} = 0; while {guard} < {limit} {{ {body} {guard} += 1; }}") + } + + fn for_loop(&mut self) -> String { + let item = self.name("it"); + let iterable = if self.rng.chance(2) { format!("0..{}", 1 + self.rng.below(4)) } else { self.array() }; + + self.vars.push(item.clone()); + self.loops += 1; + let body = self.block(); + self.loops -= 1; + self.vars.pop(); + + format!("for {item} in {iterable} {body}") + } + + fn switch(&mut self) -> String { + let subject = self.expression(); + let mut arms = Vec::new(); + for value in 0..1 + self.rng.below(3) { + arms.push(format!("{value} => {}", self.expression())); + } + if self.rng.chance(3) { + arms.push(format!("0..={} => {}", 9, self.expression())); + } + arms.push(format!("_ => {}", self.expression())); + format!("switch {subject} {{ {} }};", arms.join(", ")) + } + + fn try_catch(&mut self) -> String { + let mut body = self.statements(); + // A `throw` inside, most of the time: rhai's optimizer replaces a `try` + // whose body is pure with a plain block, so a pure body would test the + // optimizer rather than the handler. + if !self.rng.chance(3) { + let thrown = self.expression(); + body.push_str(&format!("throw {thrown};")); + } + let catch = self.block(); + if self.rng.chance(2) { + format!("try {{ {body} }} catch (e) {catch}") + } else { + format!("try {{ {body} }} catch {catch}") + } + } + + fn variable(&mut self) -> Option { + if self.vars.is_empty() { + return None; + } + Some(self.vars[self.rng.below(self.vars.len())].clone()) + } + + /// An array literal, empty about a quarter of the time. + /// + /// Empty matters on its own: it contributes no size check, so nesting one + /// inside a literal that does is what catches a running total being popped + /// by the wrong literal. + fn array(&mut self) -> String { + let items: Vec = (0..self.rng.below(4)).map(|_| self.element()).collect(); + format!("[{}]", items.join(", ")) + } + + fn map(&mut self) -> String { + let entries: Vec = (0..self.rng.below(4)).map(|n| format!("k{n}: {}", self.element())).collect(); + format!("#{{ {} }}", entries.join(", ")) + } + + /// What goes inside a literal. + /// + /// Sometimes another literal, so nesting is reached — an all-constant one + /// is folded away by the optimizer before the compiler sees it, and only a + /// computed element keeps the literal alive to run time. + fn element(&mut self) -> String { + if self.depth < MAX_DEPTH && self.rng.chance(4) { + self.depth += 1; + let out = if self.rng.chance(2) { self.array() } else { self.map() }; + self.depth -= 1; + return out; + } + self.atom() + } + + /// A leaf: cheap, and always available however deep the nesting has got. + fn atom(&mut self) -> String { + match self.rng.below(8) { + 0..=2 => format!("{}", self.rng.below(64)), + // A float literal is not syntax under `no_float` — rhai reads the + // `.` as a property access — so the whole script would fail to + // parse and test nothing. + #[cfg(not(feature = "no_float"))] + 3 => format!("{}.{}", self.rng.below(8), self.rng.below(8)), + 4 => format!("\"s{}\"", self.rng.below(8)), + 5 => self.rng.one_of(&["true", "false"]).to_string(), + 6 => "'c'".to_string(), + _ => self.variable().unwrap_or_else(|| "1".to_string()), + } + } + + /// Something the parser will accept where a boolean is required. + fn condition(&mut self) -> String { + if self.depth >= MAX_DEPTH { + return self.rng.one_of(&["true", "false"]).to_string(); + } + self.depth += 1; + let out = match self.rng.below(8) { + 0 => "true".to_string(), + 1 => "false".to_string(), + 2 => format!("(!{})", self.condition()), + 3 => { + let lhs = self.condition(); + let op = self.rng.one_of(&["&&", "||"]); + format!("({lhs} {op} {})", self.condition()) + } + _ => { + let lhs = self.expression(); + let op = self.rng.one_of(COMPARISONS); + format!("({lhs} {op} {})", self.expression()) + } + }; + self.depth -= 1; + out + } + + pub fn expression(&mut self) -> String { + if self.depth >= MAX_DEPTH { + return self.atom(); + } + self.depth += 1; + let out = self.compound(); + self.depth -= 1; + out + } + + fn compound(&mut self) -> String { + match self.rng.below(14) { + 0..=3 => self.atom(), + 4 | 5 => { + let lhs = self.expression(); + let op = self.rng.one_of(BINARY); + format!("({lhs} {op} {})", self.expression()) + } + 6 => format!("(-{})", self.expression()), + // `!` takes a condition rather than any expression: rhai rejects + // `!9` at parse time the same way it rejects `if 9`. + 7 => format!("(!{})", self.condition()), + 8 => self.array(), + 9 => self.map(), + 10 => { + // A chain over each of the three `Root` variants: a declared + // variable, a name that is declared nowhere — which is what a + // caller would have supplied, and here resolves to nothing — + // and a literal. + let root = match self.variable() { + _ if self.rng.chance(6) => "absent".to_string(), + Some(var) if self.rng.chance(2) => var, + _ => self.array(), + }; + match self.rng.below(3) { + 0 => format!("{root}[{}]", self.rng.below(4)), + 1 => format!("{root}.{}()", self.rng.one_of(METHODS)), + _ => format!("{root}.a"), + } + } + 11 if !self.interpolating => { + self.interpolating = true; + let segment = self.expression(); + self.interpolating = false; + format!("`i{}{{{segment}}}`", self.rng.below(8)) + } + 11 => self.atom(), + 12 => self.call(), + _ => { + // A closure, called directly. Handing one to a native is the + // shape with the known divergence, so it is not generated. + let param = self.name("c"); + self.vars.push(param.clone()); + let body = self.expression(); + self.vars.pop(); + format!("(|{param}| {body}).call({})", self.atom()) + } + } + } + + fn call(&mut self) -> String { + if self.rng.chance(4) { + return self.native_call(); + } + if self.functions.is_empty() { + return self.atom(); + } + let (name, arity) = self.functions[self.rng.below(self.functions.len())].clone(); + let args: Vec = (0..arity).map(|_| self.atom()).collect(); + format!("{name}({})", args.join(", ")) + } + + /// A native in function-call style, with a variable in first position + /// wherever there is one to use — see [`NATIVES`]. + fn native_call(&mut self) -> String { + let (name, arity) = NATIVES[self.rng.below(NATIVES.len())]; + let receiver = self.variable().unwrap_or_else(|| self.array()); + let rest: Vec = (1..arity).map(|_| self.atom()).collect(); + + if rest.is_empty() { + return format!("{name}({receiver})"); + } + format!("{name}({receiver}, {})", rest.join(", ")) + } +} diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs new file mode 100644 index 000000000..fd2eddf2e --- /dev/null +++ b/tests/grain/corpus/mod.rs @@ -0,0 +1,596 @@ +//! Scripts the VM must agree with rhai on. +//! +//! Weighted towards the places a bytecode VM is most likely to drift from a +//! tree walker rather than towards breadth: scope discipline, the error-based +//! unwinding rhai uses for `return`/`break`/`throw`, and the lvalue forms that +//! cannot be expressed as a plain `&mut` and so need explicit write-back. +//! +//! Everything here currently runs as a single `EvalAst` residual, so passing is +//! expected. That is the point — it pins the baseline, and it exercises the +//! runtime-state setup (function library, module resolver, source name, +//! `return`/`exit` mapping) which is real code that can be wrong today. + +use rhai::INT; + +// Only `tests/fuzz.rs` and the `generated` fuzz target use this; the other +// harnesses take the module for its cases. +#[allow(dead_code)] +pub mod generate; + +pub struct Case { + pub name: &'static str, + pub source: &'static str, +} + +/// A host type with a getter, a setter, an indexer and both kinds of method. +/// +/// Registered so the corpus can reach the one part of the chain walker that is +/// conservative rather than exact. Arrays and maps hand out references, so a +/// mutation partway down a chain lands in them and nothing needs writing back. +/// A getter hands back a *value*, so `w.inner.level = 1` mutates a temporary +/// and the setter is the only way home — and rhai decides whether to call it +/// from `func.is_method()`, which the VM cannot see and therefore approximates. +/// Without a host type in the engine, nothing here is ever exercised. +/// Held as `INT` rather than `i64` throughout: `only_i32` narrows the script +/// integer, and a host type registered against the wider one would take a type +/// no script under that build can produce. +#[derive(Debug, Clone, Default)] +pub struct Widget { + pub level: INT, + pub cells: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct Holder { + pub inner: Widget, +} + +fn out_of_range(index: INT, len: usize) -> Box { + Box::new(rhai::EvalAltResult::ErrorArrayBounds(len, index, rhai::Position::NONE)) +} + +/// The engine both sides of the differential run against. +pub fn engine() -> rhai::Engine { + let mut engine = rhai::Engine::new(); + + engine + .register_type_with_name::("Widget") + .register_fn("widget", |level: INT| Widget { level, cells: vec![10, 20, 30] }) + .register_get_set("level", |w: &mut Widget| w.level, |w: &mut Widget, v: INT| w.level = v) + // Returning an error rather than panicking, because a panic in a + // registered function takes the test process with it. + .register_indexer_get_set( + |w: &mut Widget, i: INT| -> Result> { w.cells.get(i as usize).copied().ok_or_else(|| out_of_range(i, w.cells.len())) }, + |w: &mut Widget, i: INT, v: INT| -> Result<(), Box> { + let len = w.cells.len(); + *w.cells.get_mut(i as usize).ok_or_else(|| out_of_range(i, len))? = v; + Ok(()) + }, + ) + // Takes the receiver by reference, so rhai counts it as a method and + // writes a temporary back afterwards. + .register_fn("bump", |w: &mut Widget| w.level += 1) + // Reads only. Whether rhai still writes back after one of these is + // exactly the question this corpus is here to settle. + .register_fn("doubled", |w: &mut Widget| w.level * 2) + // Mutates and *then* fails. Rhai reaches a chain root through a live + // reference, so the mutation has already landed by the time the error + // propagates; nothing else here can tell a write-back that happens from + // one that is skipped because the walk raised. + .register_fn("bump_then_fail", |w: &mut Widget| -> Result<(), Box> { + w.level += 1; + Err("bump_then_fail".into()) + }); + + engine + .register_type_with_name::("Holder") + .register_fn("holder", |level: INT| Holder { inner: Widget { level, cells: vec![1, 2, 3] } }) + .register_get_set("inner", |h: &mut Holder| h.inner.clone(), |h: &mut Holder, w: Widget| h.inner = w); + + engine +} + +const fn case(name: &'static str, source: &'static str) -> Case { + Case { name, source } +} + +/// Whether a corpus case exercises anything on this build. +/// +/// A restriction feature removes the syntax outright — rhai will not parse a +/// capturing closure under `no_closure`, or a float literal under `no_float` — +/// so the case tests nothing here, and both sides agreeing on the parse failure +/// would be an empty agreement rather than a passing one. +/// +/// Lives here rather than in one harness because every harness that walks +/// [`CASES`] needs the same answer. +#[must_use] +pub fn applies_to_this_build(name: &str) -> bool { + #[cfg(feature = "no_closure")] + if name.starts_with("closure_") || name.starts_with("is_shared") { + return false; + } + #[cfg(feature = "no_module")] + if name.starts_with("import_") || name.starts_with("export_") { + return false; + } + // `unchecked` removes the arithmetic guards, so `1 / 0` panics inside + // rhai's own built-in rather than raising — there is no behaviour left for + // the two sides to agree on, and the case would take the process with it. + #[cfg(feature = "unchecked")] + if matches!(name, "error_divide_by_zero" | "error_temp_root_index_runs_first") { + return false; + } + // No shared prefix to key on: a float literal is incidental to most of + // these, which are about interpolation, ranges and operator errors. + #[cfg(feature = "no_float")] + if matches!( + name, + "float_arithmetic" | "mixed_numeric" | "interpolation_of_every_type" | "switch_float_in_range" | "error_operator_undefined_for_types" | "error_op_assign_undefined_for_types" + ) { + return false; + } + let _ = name; + true +} + +pub const CASES: &[Case] = &[ + // --- values and operators ------------------------------------------- + case("int_arithmetic", "let a = 7; let b = 3; a * b - a / b + a % b"), + case("float_arithmetic", "let a = 7.5; let b = 0.5; a * b + a / b"), + case("mixed_numeric", "1 + 2.5"), + case("comparison_chain", "let a = 5; a > 1 && a < 10 || a == 5"), + case("bitwise", "let a = 0b1010; (a & 0b0110) | (a ^ 0b1111) << 2"), + case("string_ops", r#"let s = "hello"; s + " " + "world" + s.len"#), + case("string_interpolation", r#"let n = 42; `answer is ${n} and ${n * 2}`"#), + // Every segment type goes through a different arm of rhai's rendering: + // strings skip dispatch entirely, unit renders empty, and a container + // gets its debug-ish form. + case("interpolation_of_every_type", r#"let s = "x"; let n = 1; let f = 1.5; let b = true; let u = (); `${s}|${n}|${f}|${b}|${u}|`"#), + case("interpolation_of_containers", r#"let a = [1, 2]; let m = #{ k: 1 }; `${a}|${m}`"#), + // A host type with no `to_string` registered falls back to the mapped + // type name rather than to `Debug`. + case("interpolation_of_host_type", r#"let w = widget(3); `w=${w}`"#), + case("char_ops", r#"let c = 'a'; c.to_upper()"#), + case("unit_value", "()"), + // --- containers ------------------------------------------------------- + case("array_literal", "let a = [1, 2, 3]; a[0] + a[1] + a[2]"), + case("array_methods", "let a = [3, 1, 2]; a.sort(); a"), + case("map_literal", r#"let m = #{ a: 1, b: 2 }; m.a + m.b"#), + // The one above is all-constant, so the optimizer folds it and the map + // never gets built at run time. These do get built: rhai keeps a template + // holding every key and fills the computed ones in afterwards. + case("map_computed_value", "let v = 7; let m = #{ a: v, b: 2 }; m.a + m.b"), + case("map_all_computed", "let v = 7; let w = 8; #{ a: v, b: w }"), + case("map_computed_nested", "let v = 7; #{ outer: #{ inner: v } }.outer.inner"), + case("map_computed_in_array", "let v = 7; [#{ a: v }, #{ a: 2 }]"), + // An empty literal inside one that is not. It contributes no size check of + // its own, so it must not consume the enclosing literal's running total. + case("empty_literals_nested_in_computed_ones", "let v = 7; [v, [], #{}, v]"), + case("empty_map_nested_in_a_computed_map", "let v = 7; #{ a: v, b: #{}, c: [] }"), + // The value is a call, so the order it runs in relative to the rest of the + // literal is observable. + case("map_computed_order", r#"let log = ""; fn note(s, c) { s + c } let m = #{ a: note("", "x"), b: note("", "y") }; m.a + m.b"#), + case("nested_containers", r#"let m = #{ xs: [1, 2, #{ y: 3 }] }; m.xs[2].y"#), + // --- control flow ----------------------------------------------------- + case("if_else", "let a = 5; if a > 3 { \"big\" } else { \"small\" }"), + case("while_loop", "let i = 0; let s = 0; while i < 5 { s += i; i += 1; } s"), + case("do_while", "let i = 0; do { i += 1; } while i < 3; i"), + case("do_until", "let i = 0; do { i += 1; } until i >= 3; i"), + case("loop_break_value", "let i = 0; loop { i += 1; if i > 4 { break i * 10; } }"), + case("continue_skips", "let s = 0; for i in 0..10 { if i % 2 == 0 { continue; } s += i; } s"), + case("for_range", "let s = 0; for i in 0..5 { s += i; } s"), + case("for_array", "let s = 0; for x in [10, 20, 30] { s += x; } s"), + case("for_with_counter", "let s = 0; for (x, i) in [10, 20, 30] { s += x * i; } s"), + // The loop variable is pushed once and mutated in place rather than + // re-pushed each iteration (eval/stmt.rs:752); a VM that re-pushes would + // leave the scope a different depth. + case("for_loop_var_not_leaked", "let x = 99; for x in 0..3 { } x"), + case("nested_loops_break", "let s = 0; for i in 0..3 { for j in 0..3 { if j == 2 { break; } s += 1; } } s"), + // An empty body is a separate path in rhai that never touches the loop + // variable or the counter (`eval/stmt.rs:719`). + case("for_empty_body", "let s = 0; for i in 0..5 { } s"), + // `return` out of a `for` skips the exhaustion path, so the iterator and + // both loop variables have to go with the frame. + case("for_return_from_body", "fn find(xs) { for (x, i) in xs { if x > 1 { return i; } } -1 } find([1, 2, 3])"), + // A `break` out of a `while` nested in a `for` must drop nothing, and out + // of the `for` must drop one — the two are easy to get the wrong way round. + case("for_around_while_break", "let s = 0; for i in 0..3 { let j = 0; while true { j += 1; if j > 2 { break; } s += 1; } } s"), + // Iterating a shared cell walks a snapshot, because rhai flattens the + // iterable before asking for an iterator (`eval/stmt.rs:677`). + case("for_over_captured_array", "let a = [1, 2, 3]; { let f = || a; } let s = 0; for x in a { s += x; } s"), + // --- switch ----------------------------------------------------------- + case("switch_literal", "let x = 2; switch x { 1 => \"one\", 2 => \"two\", _ => \"other\" }"), + case("switch_range", "let x = 42; switch x { 0..=9 => \"small\", 10..=99 => \"medium\", _ => \"large\" }"), + // A failing guard must fall through to the next matching case, not to the + // default, so both single-digit arms are needed to tell those apart. + case("switch_guard", "let x = 5; switch x { 0..=9 if x % 2 == 1 => \"odd digit\", 0..=9 => \"even digit\", _ => \"big\" }"), + case("switch_default_only", "switch 999 { 1 => \"a\", _ => \"fallback\" }"), + // Two case values, one arm: the table has two entries pointing at one + // body, which a compiler emitting a body per entry would duplicate. + case("switch_shared_body", "let x = 2; switch x { 1 | 2 => \"low\", 3 => \"three\", _ => \"other\" }"), + // The rule that reads like a bug and is not: a case value that matched but + // whose guard declined goes to the *default*, never on to the ranges + // (eval/stmt.rs:544). Without the range arm here the two are the same + // answer and the case proves nothing. + case("switch_declined_case_skips_ranges", "let f = false; let x = 1; switch x { 1 if f => \"guarded\", 0..=5 => \"range\", _ => \"default\" }"), + // No `_` arm at all, so the miss has to produce unit from nowhere. + case("switch_no_default", "let x = 9; switch x { 1 => \"a\" }"), + case("switch_string", "let s = \"b\"; switch s { \"a\" => 1, \"b\" => 2, _ => 0 }"), + // A range arm covers the reals between its bounds, so a float lands in one + // even though the bounds are integers. + case("switch_float_in_range", "let x = 5.5; switch x { 0..10 => \"in\", _ => \"out\" }"), + // Hashing a host type panics, so the subject has to be checked before it + // reaches a hasher — and must still find the default. + case("switch_unhashable_subject", "let w = widget(3); switch w { 1 => \"int\", _ => \"other\" }"), + // A shared value is not hashable either, so rhai skips the cases *and* the + // ranges and goes straight to the default — however well the value would + // otherwise have matched. Reading the subject through its cell would hide + // that, which is why the subject is loaded unflattened. + case("switch_on_a_shared_subject_takes_the_default", r#"let v = 0; { let f = || v; } switch v { 0 => "case", _ => "default" }"#), + case("switch_range_on_a_shared_subject_takes_the_default", r#"let v = 5; { let f = || v; } switch v { 0..=9 => "range", _ => "default" }"#), + // The same shape before anything shares it, so the pair says the difference + // is the sharing rather than the switch. + case("switch_on_an_unshared_subject_matches", r#"let v = 0; switch v { 0 => "case", _ => "default" }"#), + // An arm body is a block: it declares, and it has to leave the scope the + // depth it found it — which only shows up in something that reads a local + // afterwards. + case("switch_block_body_scope", "let x = 1; let y = 0; switch x { 1 => { let z = 5; y = z * 2 }, _ => () } y"), + // A jump out of an arm and out of the switch, which is where the operand + // stack most plausibly ends up a different depth on the two paths. + // A range is a host type as far as `Dynamic` is concerned, and + // `is_hashable` says no to those — even though `Hash for Dynamic` would in + // fact hash a range (types/dynamic.rs:465). So rhai never matches a range + // *subject* against anything, and neither may the VM: mirroring the gate + // matters more than being clever about it. + case("switch_range_subject_never_matches", "let r = 0..5; switch r { 0..5 => \"same\", _ => \"no\" }"), + case("switch_break_from_loop", "let s = 0; let i = 0; while i < 10 { switch i { 3 => break, _ => () } s += 1; i += 1; } s"), + // --- blocks used for their value --------------------------------------- + // Rhai wraps a block in `Expr::Stmt` wherever a value is wanted, so these + // are one construct in three disguises. Each declares inside the block, so + // a lowering that forgot to rewind would leave the scope a different depth + // and every slot after it would name the wrong variable. + case("let_from_switch", "let x = 2; let y = switch x { 1 => \"one\", 2 => \"two\", _ => \"other\" }; y"), + case("let_from_if", "let c = true; let y = if c { let a = 1; a } else { let b = 2; b }; y + 10"), + case("let_from_block", "let a = 3; let y = { let z = a; z * 2 }; y"), + // A block among a call's arguments, where the scope grows while operands + // are already on the stack. + case("block_as_argument", "fn add(a, b) { a + b } let n = 2; add({ let t = n; t + 1 }, 10)"), + // --- scoping ---------------------------------------------------------- + case("shadowing_nested", "let x = 1; { let x = 2; { let x = 3; } } x"), + case("block_scope_discarded", "let x = 1; { let y = 2; x += y; } x"), + case("const_read", "const K = 10; K * 2"), + // --- functions -------------------------------------------------------- + case("fn_call", "fn add(a, b) { a + b } add(2, 3)"), + // Kept shallow deliberately: rhai's default call-depth limit is far lower + // in debug builds than in release, and this case is about recursion working + // at all, not about the limit. The limit gets its own case. + case("fn_recursion", "fn fib(n) { if n < 2 { n } else { fib(n - 1) + fib(n - 2) } } fib(6)"), + case("fn_early_return", "fn f(x) { if x > 0 { return \"pos\"; } \"nonpos\" } f(1) + f(-1)"), + // A script method mutating its receiver: `this` is bound by reference, so + // the write has to land back in the caller's variable. + case("fn_mutating_method", "fn double() { this *= 2; } let v = 21; v.double(); v"), + case("top_level_return", "let x = 5; if x > 0 { return x * 2; } 0"), + // A script function this compiler could not lower still runs — rhai finds + // it in `global.lib` — and it must run in a scope of its own. Handing it + // this frame's would let the body read the caller's locals, where rhai + // gives it an empty one (`func/call.rs:1476`), and the read is the whole + // difference: the walker cannot find `secret` and a VM that leaked its + // scope answers with 42. + // + // `this` is what leaves the body unlowerable, and calling it by name + // rather than as a method is what routes it through generic dispatch. + case("error_a_skipped_function_cannot_see_the_caller", "fn peek(k) { let seen = secret; this } let secret = 42; peek(1)"), + // --- a variable in first-argument position ------------------------------ + // Rhai rewrites `f(x, ..)` into `x.f(..)` so a `&mut` first parameter + // mutates the variable (`func/call.rs:1434`). These are the same calls the + // chain cases make in method syntax, and they have to mean the same thing. + case("call_style_mutating_native", "let a = [1]; push(a, 2); a"), + case("call_style_mutating_host_type", "let w = widget(4); bump(w); w.level"), + case("call_style_pure_native", "let a = [1, 2]; len(a)"), + // Rhai reads the variable *after* the other arguments, so an argument that + // writes to it is seen. Two shapes of write, because one goes through the + // rewrite itself and the other does not. + case("call_style_argument_writes_the_receiver", "let a = [1]; push(a, { push(a, 9); 2 }); a"), + case("call_style_argument_replaces_the_receiver", "let a = [1]; push(a, { a = [7]; 2 }); a"), + // The receiver appearing again among the arguments, which is where a live + // reference into the scope would be most likely to show. It does not: the + // later argument was read and flattened before the reference was taken, so + // it is a copy of what the receiver held then. + case("call_style_receiver_is_also_an_argument", "let a = [1]; push(a, a); a"), + case("call_style_receiver_twice_over", "let a = [1]; insert(a, 0, a); a"), + // Neither of these can be handed out by reference, so both are passed by + // value and the mutation is discarded (`func/call.rs:1449-1454`). + case("call_style_constant_receiver", "const a = [1]; push(a, 2); a"), + // The closure is made in a block so the scope the two sides are compared on + // does not end up holding a pointer, which they render differently on + // purpose — see `a_closure_pointer_is_late_bound` in `tests/scope.rs`. + case("call_style_shared_receiver", "let a = [1]; { let f = || a.len(); } push(a, 2); a"), + // A script function copies its first argument whichever way it arrives, so + // the rewrite is invisible here — which is the thing to pin. + case("call_style_script_fn", "fn bump_it(x) { x += 1; x } let n = 3; bump_it(n); n"), + // The receiver resolves last, so a missing one is reported after a missing + // argument rather than before it. + case("error_receiver_resolves_after_arguments", "nosuch(receiver, argument)"), + case("error_receiver_not_found", "let ok = 1; nosuch(missing, ok)"), + // Dispatch still fails at the call, not at the variable that reached it. + case("error_no_function_for_the_receiver", "let a = [1]; nosuch(a, 2)"), + // An error a native *returned*, which rhai positions at the call site like + // everything else dispatch produces (`func/call.rs:413`). Both argument + // shapes, because one goes through the rewrite and the other does not. + case("error_returned_by_a_native", r#"parse_int("zz")"#), + case("error_returned_by_a_native_by_reference", r#"let s = "zz"; parse_int(s)"#), + // --- closures --------------------------------------------------------- + // Function pointers are invoked via `.call()`; `f(5)` would look for a + // function literally named `f`. + // The closure is kept inside a block in all of these. Not incidental: the + // pointer we build is late-bound where rhai's is early-bound, so rhai + // renders it `Fn*+("anon$..")` and we render it `Fn("anon$..")`. That + // difference is the price of not shipping an AST body, it is script- + // visible, and `a_closure_pointer_is_late_bound` is where it is pinned — + // so these cases test the capture rather than re-testing the rendering. + case("closure_capture_read", "let n = 10; let r = 0; { let f = |x| x + n; r = f.call(5); } r"), + // Capture is by shared cell, so the mutation must be visible outside. + case("closure_capture_mutate", "let n = 0; { let f = || n += 1; f.call(); f.call(); } n"), + // In a block for the same reason the closure cases are: what the pointer + // *does* matches, what it renders as does not. + case("fn_ptr_call", "fn triple(x) { x * 3 } let r = 0; { let f = Fn(\"triple\"); r = f.call(4); } r"), + // The same through a name that is not a constant, so rhai's optimizer + // cannot fold it into a pointer carrying an environment. + case("fn_ptr_from_dynamic_name", "fn triple(x) { x * 3 } let n = \"trip\" + \"le\"; let f = Fn(n); f.call(4)"), + case("fn_ptr_curried", "fn add(a, b) { a + b } let n = \"a\" + \"dd\"; let f = Fn(n).curry(10); f.call(5)"), + // A pointer to a native function goes to rhai's own dispatch rather than + // to a chunk of ours. + case("fn_ptr_to_native", "let n = \"ab\" + \"s\"; let f = Fn(n); f.call(-7)"), + // Deliberately absent: `let x = 1; x.call(2)`. That is not an error in + // rhai — a non-pointer target means the *argument* is the pointer and the + // target is `this` — and the VM reproduces the behaviour, but not the + // position. Rhai blames the argument there and the call everywhere else, + // and one instruction has one position-table entry; using the argument's + // was measured to move the divergence onto the common path instead of + // removing it. A pool of positions would fix it and would not be + // strippable. + case("error_fn_ptr_unknown_name", "let n = \"no\" + \"pe\"; let f = Fn(n); f.call(1)"), + // `Fn` and `curry` read their first argument and blame everything they can + // then complain about on *it* rather than on the call — a name that is not + // a string, a string that is not an identifier, a first argument that is + // not a pointer (`func/call.rs:1217`, `:1220`, `:1232`). + case("error_fn_ptr_from_a_non_string", "Fn(())"), + case("error_fn_ptr_from_an_unusable_name", r#"Fn("not an identifier!")"#), + case("error_curry_of_a_non_pointer", "curry(1, 2)"), + // Capturing a variable turns its slot into a shared cell, which changes + // what every later read and write of that slot means. Writing the slot + // instead of writing *through* it severs the closure silently — the value + // is right and the aliasing is dead — so this needs a write after the + // capture to catch it. + case("closure_shared_write", "let x = 1; let r = 0; { let f = || x; x = 42; r = f.call(); } r"), + case("closure_shared_op_assign", "let x = 1; let r = 0; { let f = || x; x += 41; r = f.call(); } r"), + // The same cell as the root of a chain. `get_indexed_mut` refuses a shared + // value outright, so walking one takes the host down rather than returning + // an error (`eval/chaining.rs:461`). + case("closure_shared_chain_root", "let a = [1, 2, 3]; { let f = || a[0]; } a[1] = 20; a[1]"), + // rhai answers this syntactically and registers no function for it, so a + // lowered call would fail to resolve where the walker returns a bool. + case("is_shared_after_capture", "let x = 1; let r = false; { let f = || x; r = is_shared(f); } [is_shared(x), r]"), + case("closure_in_map", "[1, 2, 3].map(|x| x * 2)"), + case("closure_in_filter", "[1, 2, 3, 4].filter(|x| x % 2 == 0)"), + // --- chained lvalues -------------------------------------------------- + // Each of these needs a different `Target` variant and its write-back. + case("index_assign_array", "let a = [1, 2, 3]; a[1] = 99; a"), + case("index_assign_nested", "let m = #{ xs: [1, 2, 3] }; m.xs[2] = 42; m.xs"), + case("property_assign_deep", "let m = #{ a: #{ b: #{ c: 1 } } }; m.a.b.c = 7; m.a.b.c"), + case("map_autovivify", "let m = #{}; m.fresh = 1; m"), + // The other half of it: only a write creates a key. Reading one that is + // not there gives unit and must leave the map alone — the map is returned + // so the test can see whether it grew. + case("map_read_of_absent_key_does_not_create_it", "let m = #{ a: 1 }; let r = m.b; [m, r]"), + case("map_read_absent_through_a_chain", "let m = #{ a: #{} }; let r = m.a.b; [m, r]"), + // Walking through an absent key reaches a detached unit, so the write has + // nowhere to land and rhai says so rather than creating the path. + case("error_map_write_through_an_absent_key", "let m = #{}; m.a.b = 1; m"), + // A closure holds the same cell, so a key invented by a read would be + // visible from outside the expression that invented it. + case("map_read_of_absent_key_is_not_visible_to_a_closure", "let m = #{}; let r = 0; { let f = || m; r = m.zz; } [m, r]"), + case("op_assign_indexed", "let a = [1, 2, 3]; a[0] += 10; a"), + case("bitfield_assign", "let x = 0; x[2] = true; x"), + case("string_char_assign", r#"let s = "hello"; s[0] = 'H'; s"#), + case("string_slice_read", r#"let s = "hello world"; s[0..5]"#), + // The inclusive form is a different `TypeId` and a different pool tag, so + // one does not cover the other. A string rather than an array, because + // rhai indexes arrays with integers only and slices them with `extract`. + case("string_slice_inclusive", r#"let s = "hello world"; s[6..=9]"#), + // --- chains rooted at something that is not a variable ------------------ + // Rhai evaluates the root into a temporary and walks that + // (`eval/chaining.rs:561-571`), so there is no scope entry behind it and + // nothing is written back. One case per root shape, because each reaches a + // different `Target`. + case("temp_root_array_method", "[3, 1, 2].len()"), + case("temp_root_array_index", "[10, 20, 30][1]"), + case("temp_root_string", r#""hello".to_upper()"#), + case("temp_root_map_property", "#{ a: 1, b: 2 }.b"), + case("temp_root_call", "fn make() { [1, 2, 3] } make().len()"), + case("temp_root_parenthesised", "let a = 1; let b = 2; (a + b).to_string()"), + case("temp_root_nested", "[[1, 2], [3, 4]][1][0]"), + // A mutating method on a temporary. The mutation has nowhere to land, and + // the point is that both sides agree it is discarded rather than one of + // them inventing a place to put it. + case("temp_root_mutating_method", "let a = [1, 2, 3]; [a.len()].push(9)"), + case("temp_root_host_mutates", "widget(4).bump()"), + case("temp_root_host_pure", "widget(4).doubled()"), + // Order, which is the part that is not obvious: rhai collects a chain's + // indices *before* it evaluates what they apply to. Both halves fail, so + // the position in the reported error is which one ran first. + // An operator with no implementation for the types it got. The corpus + // reaches `ErrorFunctionNotFound` through a named call elsewhere, which is + // a different dispatch path and positions itself differently. + case("error_operator_undefined_for_types", "let a = 1.0; a + #{ b: 1 }"), + // A chain step that fails, one per kind. Rhai blames the step rather than + // the chain, and a chain is one instruction with one position-table entry, + // so these are what make each step carry its own. + case("error_property_on_a_variable", "let x = 1; x.a"), + case("error_property_on_a_temporary", "[1, 2].a"), + case("error_method_on_a_variable", "let x = 1; x.to_upper()"), + case("error_property_deep_in_a_chain", "let m = #{ a: #{} }; m.a.b.c"), + // The op-assign form, which falls back to the plain operator when no + // `+=` is registered and used to lose the position on the way. + case("error_op_assign_undefined_for_types", "let a = 1.0; a += #{ b: 1 }; a"), + case("error_temp_root_index_runs_first", "let z = 0; [1 / z][9 / z]"), + // Two positions belong to an index step, not one: the index expression, + // which an out-of-bounds is blamed on, and the `[` in front of it, which + // indexing something unindexable is blamed on. They only come apart in a + // chain of more than one step — here `n[0]` bit-indexes an integer and + // yields a bool, and rhai names the *second* `[`. + case("error_index_into_an_unindexable_step", "let n = 0; n[0][5]"), + case("error_index_into_an_unindexable_step_deep", "let m = #{ a: 1 }; m.a[0][5]"), + // --- what an escaping error leaves in the scope ------------------------- + // Rhai rewinds a block whether it is left normally or by a throw, and + // rewinds nothing at the top level. The comparison that matters in all of + // these is the leftover scope rather than the error. + case("throw_from_a_block_rewinds_it", "let a = 1; { let b = 2; throw 3; }"), + case("throw_from_a_for_body_drops_the_loop_var", "let a = 1; for i in 0..3 { throw i; }"), + case("throw_from_a_while_body_drops_its_locals", "let a = 1; let n = 0; while n < 3 { let b = n; throw b; }"), + // A catch block is a block too, and its variable is the one rhai pushes + // rather than the script. + case("throw_from_a_catch_drops_the_catch_var", "let a = 1; try { throw 2; } catch (e) { throw e; }"), + case("throw_from_a_nested_block_drops_every_level", "let a = 1; { let b = 2; { let c = 3; for i in 0..2 { throw i; } } }"), + // The frame boundary: a function's own locals go with its scope, and the + // caller's top-level ones stay. + case("throw_from_a_function_leaves_the_caller_top_level_alone", "fn boom() { let inner = 9; throw inner; } let a = 1; { let b = 2; boom(); }"), + // Nothing to rewind, which is the case a floor set too low would break. + case("throw_at_the_top_level_keeps_what_ran", "let a = 1; let b = 2; throw 3;"), + case("error_temp_root_out_of_bounds", "[1, 2, 3][99]"), + // --- errors ----------------------------------------------------------- + // Compared by variant and position, so a VM that reports the right failure + // at the wrong place still fails the test. + case("error_unknown_variable", "let a = 1; a + nonexistent"), + // Positions on call failures are set by different code paths depending on + // whether the callee was found, so both need pinning. + case("error_unknown_function", "let a = 1; no_such_function(a)"), + case("error_wrong_arity", "fn f(a, b) { a } f(1)"), + case("error_array_bounds", "let a = [1, 2]; a[10]"), + // Rhai maps the *expected* type through its registered names and leaves the + // *actual* one raw, so a range guard reports `core::ops::range::Range` + // rather than the `range` the same engine prints everywhere else. Mapping + // both is the obvious mistake, and only a type with a registered name shows + // it up. + case("error_condition_is_a_range", "if 0..1 { 1 } else { 2 }"), + case("error_condition_is_a_host_type", "let w = widget(1); while w { 1 }"), + case("error_type_mismatch", r#"let a = 1; a + "string" + [1]"#), + case("error_divide_by_zero", "let a = 1; a / 0"), + case("throw_value", "throw 42"), + case("throw_in_fn", "fn f() { throw \"boom\"; } f()"), + // --- try / catch ------------------------------------------------------ + case("try_catch_value", "try { throw 7; } catch (e) { e * 2 }"), + case("try_catch_native_error", "try { let a = [1]; a[9] } catch (e) { e.message != () }"), + case("try_catch_rethrow", "try { try { throw 1; } catch { throw; } } catch (e) { e }"), + // `return` unwinds as an error but must pass straight through a catch. + case("try_catch_does_not_swallow_return", "fn f() { try { return 1; } catch { return 2; } } f()"), + case("try_catch_no_error", "try { 5 } catch { 6 }"), + // The catch block's value is discarded — the statement is unit on the + // caught path and the try block's value otherwise (`eval/stmt.rs:863`). + case("try_catch_discards_its_value", "try { throw 1; } catch { 99 }"), + // A jump out of a `try` skips the `PopHandler` the straight-line path + // would have run. Left armed, the next error is caught into a block that + // has already been left — so the second failure here must not be caught. + case("break_out_of_try_disarms_it", "let s = 0; while true { try { throw 1; } catch { break; } } try { throw 2; } catch (e) { s = e; } s"), + case("break_out_of_for_inside_try", "let s = 0; for i in 0..5 { try { if i == 2 { break; } s += i; } catch { s = -1; } } s"), + case("continue_out_of_try_inside_for", "let s = 0; for i in 0..5 { try { if i % 2 == 0 { continue; } s += i; } catch { s = -1; } } s"), + // An error out of a called function arrives wrapped in + // `ErrorInFunctionCall`, which is catchable, and `unwrap_inner` is what + // still binds the bare thrown value. + case("try_around_a_compiled_call", "fn boom() { throw 7; } try { boom(); } catch (e) { e }"), + // `return` is a pseudo error and must pass straight through a handler. + case("try_does_not_catch_return", "fn f() { try { return 1; } catch { 2 } } f()"), + // --- host types ------------------------------------------------------- + // + // The one part of the chain walker that approximates rather than + // reproduces. A getter hands back a value, so anything below it mutates a + // temporary that only the setter can put back — and rhai decides whether + // to call the setter from `func.is_method()`, which is not visible from + // outside the crate. + case("host_get", "let w = widget(4); w.level"), + case("host_set", "let w = widget(4); w.level = 9; w.level"), + case("host_op_assign", "let w = widget(4); w.level += 5; w.level"), + case("host_index_get", "let w = widget(1); w[1]"), + case("host_index_set", "let w = widget(1); w[1] = 99; w[1]"), + case("host_method_mutates", "let w = widget(4); w.bump(); w.level"), + case("host_method_pure", "let w = widget(4); w.doubled()"), + // Two levels, so the middle one is a temporary. + case("host_temp_set", "let h = holder(3); h.inner.level = 8; h.inner.level"), + case("host_temp_index_set", "let h = holder(3); h.inner[0] = 7; h.inner[0]"), + // A mutating call on a temporary: rhai writes it back, so the change + // survives. + case("host_temp_mutates", "let h = holder(3); h.inner.bump(); h.inner.level"), + // A read-only call on a temporary, which is where rhai's own flag decides + // whether a setter runs at all. + case("host_temp_pure", "let h = holder(3); h.inner.doubled()"), + case("error_host_index_bounds", "let w = widget(1); w[99]"), + // A step that mutates and then raises. The error is caught, so what is + // being compared is whether the mutation reached the variable — rhai's does, + // because it never walked a copy. + case("host_mutation_before_a_failure_survives", "let w = widget(1); try { w.bump_then_fail(); } catch(e) {} w.level"), + case("host_mutation_before_a_failure_survives_in_a_map", "let m = #{ w: widget(1) }; try { m.w.bump_then_fail(); } catch(e) {} m.w.level"), + case("host_mutation_before_a_failure_survives_in_an_array", "let a = [widget(1)]; try { a[0].bump_then_fail(); } catch(e) {} a[0].level"), + // `this`, which is a register rather than a scope entry and so is reached + // by instructions of its own. + case("this_read", "fn get() { this } let v = 7; v.get()"), + case("this_in_an_expression", "fn double() { this * 2 } let v = 21; v.double()"), + case("this_assign", "fn set() { this = 9; } let v = 1; v.set(); v"), + case("this_op_assign", "fn bump(n) { this += n; } let v = 1; v.bump(4); v"), + case("this_op_assign_on_a_string", "fn add(s) { this += s; } let v = \"a\"; v.add(\"b\"); v"), + case("this_is_the_bodys_value", "fn twice() { this + this } let v = 4; v.twice()"), + // Never inherited: a plain call from a bound body gets no receiver. + case("error_this_is_not_inherited", "fn outer() { inner() } fn inner() { this } let v = 1; v.outer()"), + case("error_this_unbound_in_call_style", "fn get() { this } get()"), + // The check precedes the right-hand side, unlike the variable arm. + case("error_this_assign_unbound_beats_a_bad_value", "fn set() { this = nosuch; } set()"), + // Chains rooted at `this`, which must write back into the caller's value. + case("this_property", "fn count() { this.n } let m = #{ n: 5 }; m.count()"), + case("this_property_assign", "fn set() { this.n = 9; } let m = #{ n: 1 }; m.set(); m.n"), + case("this_index", "fn first() { this[0] } let a = [3, 4]; a.first()"), + case("this_index_assign", "fn set() { this[0] = 9; } let a = [1, 2]; a.set(); a"), + case("this_method_step", "fn grow() { this.push(3); } let a = [1, 2]; a.grow(); a"), + case("this_host_method", "fn raise() { this.bump(); } let w = widget(4); w.raise(); w.level"), + case("this_host_property", "fn read() { this.level } let w = widget(4); w.read()"), + // A method on `this` that reaches another compiled function. + case("this_nested_method", "fn outer() { this.inner() } fn inner() { this * 2 } let v = 5; v.outer()"), + // `f(this, ..)`, which rhai rewrites to `this.f(..)` by reference. + case("this_as_first_argument", "fn grow() { push(this, 3); } let a = [1, 2]; a.grow(); a"), + case("this_as_first_argument_pure", "fn size() { len(this) } let a = [1, 2]; a.size()"), + case("this_as_a_later_argument", "fn plus(n) { n + this } let v = 1; v.plus(2)"), + // Arity excludes the receiver, so these are two different functions. + case("this_method_arity", "fn f() { 1 } fn f(x) { this + x } let v = 10; [v.f(), v.f(5)]"), + // `obj.call(f)` binds `obj` as the closure's `this` by reference, so a + // write inside the closure reaches `obj`. The operand stack only ever holds + // a copy of it, which is why the instruction carries where it came from. + // + // The pointer is scoped to a block throughout, as the other closure cases + // are: a compiled closure's `FnPtr` carries a name where rhai's carries the + // body and its environment, so one left in the scope compares unequal for a + // reason that has nothing to do with the call. + case("closure_call_on_a_local_writes_back", "let v = 21; { let f = || { this *= 2; }; v.call(f); } v"), + case("closure_call_on_a_local_inline", "let v = 21; v.call(|| { this *= 2; }); v"), + case("closure_call_on_a_local_reads", "let v = 21; let r = 0; { let f = || this * 2; r = v.call(f); } r"), + case("closure_call_mutates_an_array", "let a = [1]; { let f = || { this.push(2); }; a.call(f); } a"), + // And the receiver can be the frame's own receiver. + case("closure_call_on_this", "fn twice() { let f = || { this *= 2; }; this.call(f); } let v = 21; v.twice(); v"), + // A temporary receiver has nowhere to write back to, and rhai mutates a + // copy of it too. + case("closure_call_on_a_temporary", "let r = 0; { let f = || { this *= 2; }; r = (20 + 1).call(f); } r"), + // A native calling a pointer back against a receiver. How many arguments it + // appends beside the receiver is the native's business — `map` adds an + // index, `reduce` the running result — so no single wrapper arity is right + // and these have to stay reachable by rhai itself. + case("closure_map_binds_this", "[1, 2, 3].map(|| this * 2)"), + case("closure_filter_binds_this", "[1, 2, 3].filter(|| this > 1)"), + case("closure_for_each_binds_this", "let t = 0; [1, 2, 3].for_each(|| t += this); t"), + // And the argument form, which takes the element as a parameter instead. + case("closure_map_takes_an_argument", "[1, 2, 3].map(|x| x * 2)"), + // `type_of` has no registered implementation anywhere — rhai answers it by + // name — so it is reached through the same door every other call is. + // A constant argument is folded by the optimizer and proves nothing. + case("type_of_a_variable", "let x = 1; type_of(x)"), + case("type_of_a_container", "let a = [1]; type_of(a)"), + case("type_of_a_host_type", "let w = widget(1); type_of(w)"), + case("type_of_method_style", "let s = \"a\"; s.type_of()"), + case("type_of_a_pointer", "let r = \"\"; { let f = |x| x; r = type_of(f); } r"), +]; diff --git a/tests/grain/differential.rs b/tests/grain/differential.rs new file mode 100644 index 000000000..23728723c --- /dev/null +++ b/tests/grain/differential.rs @@ -0,0 +1,274 @@ +//! The VM must mean exactly what rhai means. +//! +//! Every corpus script is evaluated twice against the same `Engine` — once +//! through `eval_ast_with_scope`, once through the VM — and the two runs must +//! agree on the result, on the error (variant *and* position), and on the scope +//! they leave behind. +//! +//! The scope check is not incidental. Rhai evaluates a program's top-level +//! statements without rewinding, so `let` at the top level outlives the run and +//! is observable by the caller. A VM that manages its own frames could return +//! the right value and still get that wrong. + +use super::corpus; + +use rhai::grain::{Compiler, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +/// What a run produced, in a form two runs can be compared on. +/// +/// `Dynamic` and `EvalAltResult` have no `PartialEq`, so this compares their +/// `Debug` rendering. That is stricter than value equality, not looser: it +/// distinguishes `1` from `1.0`, and it includes error positions. +#[derive(Debug, PartialEq, Eq)] +struct Outcome { + result: Result, + scope: Vec<(String, String)>, +} + +fn snapshot_scope(scope: &Scope) -> Vec<(String, String)> { + scope.iter_raw().map(|(name, _, value)| (name.to_string(), format!("{value:?}"))).collect() +} + +fn run_stock(engine: &Engine, source: &str) -> Outcome { + let mut scope = Scope::new(); + let result = engine.compile(source).map_err(|err| format!("{err:?}")).and_then(|ast| { + engine + .eval_ast_with_scope::(&mut scope, &ast) + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")) + }); + + Outcome { result, scope: snapshot_scope(&scope) } +} + +fn run_vm(engine: &Engine, source: &str) -> Outcome { + let mut scope = Scope::new(); + let result = engine.compile(source).map_err(|err| format!("{err:?}")).and_then(|ast| { + let program = Compiler::new().compile(&ast); + // A program that can hand a pointer to a native has to be run the + // way such a program is meant to be run, or the comparison is + // against a configuration nobody would ship. + if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(engine).eval_with_callbacks(&mut scope, &program) + } else { + Vm::new(engine).eval_with_scope(&mut scope, &program) + } + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")) + }); + + Outcome { result, scope: snapshot_scope(&scope) } +} + +#[test] +fn vm_agrees_with_rhai() { + let engine = corpus::engine(); + + let mut failures = Vec::new(); + + for case in corpus::CASES.iter().filter(|c| applies_to_this_build(c.name)) { + let stock = run_stock(&engine, case.source); + let vm = run_vm(&engine, case.source); + + if stock != vm { + failures.push(format!( + "\n=== {} ===\n source: {}\n rhai: {:?}\n vm: {:?}\n \ + rhai scope: {:?}\n vm scope: {:?}", + case.name, case.source, stock.result, vm.result, stock.scope, vm.scope, + )); + } + } + + let applicable = corpus::CASES.iter().filter(|c| applies_to_this_build(c.name)).count(); + assert!(failures.is_empty(), "{} of {applicable} corpus scripts diverged:{}", failures.len(), failures.join(""),); +} + +/// The corpus is only worth anything if the comparison can actually fail. +/// +/// Guards against the harness silently degrading into a tautology — comparing +/// two identical code paths, or stringifying everything into the same value. +#[test] +fn harness_detects_a_real_difference() { + let engine = corpus::engine(); + + assert_ne!(run_stock(&engine, "1 + 1"), run_stock(&engine, "1 + 2"), "differing results must compare unequal",); + assert_ne!(run_stock(&engine, "1"), run_stock(&engine, "1.0"), "int and float must not compare equal",); + assert_ne!(run_stock(&engine, "let a = 1; a"), run_stock(&engine, "1"), "differing leftover scope must compare unequal",); + // `no_position` compiles positions out, so there is no such thing as the + // same error at a different one and nothing here to detect. + #[cfg(not(feature = "no_position"))] + assert_ne!(run_stock(&engine, "let a = [1]; a[9]"), run_stock(&engine, "let a = [1]; a[9]"), "the same error at a different position must compare unequal",); +} + +/// A script that does not parse compares equal on both sides for the wrong +/// reason: two identical parse errors. Such a case tests nothing, and would sit +/// in the corpus looking like coverage. +#[test] +fn every_corpus_script_parses() { + let engine = corpus::engine(); + + let broken: Vec<_> = corpus::CASES + .iter() + .filter(|case| applies_to_this_build(case.name)) + .filter_map(|case| engine.compile(case.source).err().map(|err| format!("\n {}: {err}", case.name))) + .collect(); + + assert!(broken.is_empty(), "{} corpus scripts do not parse, so they assert nothing:{}", broken.len(), broken.join(""),); +} + +/// Whether a corpus case exercises anything on this build. +/// +/// Distinct from [`MAY_FRAGMENT`], which is a tolerance: this says the syntax +/// is not in the language on this build at all. Defined beside the cases, in +/// `corpus`, because every harness that walks them needs the same answer. +use corpus::applies_to_this_build; + +/// A case that errors unintentionally is nearly as weak as one that does not +/// parse: both sides agree on the failure, and the machinery the case was +/// written to exercise never runs. Cases that mean to fail say so in the name. +#[test] +fn only_error_cases_error() { + let engine = corpus::engine(); + + let surprises: Vec<_> = corpus::CASES + .iter() + .filter(|case| !case.name.starts_with("error_") && !case.name.starts_with("throw_")) + .filter(|case| applies_to_this_build(case.name)) + .filter_map(|case| match run_stock(&engine, case.source).result { + Err(err) => Some(format!("\n {}: {err}", case.name)), + Ok(_) => None, + }) + .collect(); + + assert!(surprises.is_empty(), "{} cases fail without meaning to, so they exercise nothing:{}", surprises.len(), surprises.join(""),); +} + +/// Corpus scripts allowed to leave a fragment behind. +/// +/// Empty, and that is the claim: every script in the corpus lowers with nothing +/// left over. A case that fragments therefore fails on arrival rather than +/// quietly joining a majority, which is the point of stating it this way round. +/// +/// What legitimately belongs here: `eval`, `import`/`export`, custom syntax, +/// and `?.`. All four are the escape hatch working as intended rather than a +/// gap, and none is in the corpus. +const MAY_FRAGMENT: &[&str] = &[]; + +/// Every chunk the compiler emits must pass its own verifier. +/// +/// The check that matters is depth agreement at merge points: one branch of a +/// conditional leaving a value where the other does not is invisible until a +/// program happens to take the unlucky path, and the differential corpus only +/// covers the paths it happens to exercise. +#[test] +fn every_compiled_chunk_verifies() { + let engine = corpus::engine(); + + let broken: Vec<_> = corpus::CASES + .iter() + .filter_map(|case| { + let ast = engine.compile(case.source).ok()?; + Compiler::new().compile(&ast).verify().err().map(|err| format!("\n {}: {err:?}", case.name)) + }) + .collect(); + + assert!(broken.is_empty(), "{} chunks failed verification:{}", broken.len(), broken.join(""),); +} + +/// A chunk must declare the stack it uses, not the stack it might use. +/// +/// The lowering's own estimate is one slot per instruction, which is safe and +/// wildly loose — and it is what the VM reserves from, and what the artifact +/// records. On a device with ~12KB to spend, reserving 25 `Dynamic` slots for a +/// chunk that stacks three is the difference worth closing. +#[test] +fn every_compiled_chunk_declares_the_stack_it_uses() { + let engine = corpus::engine(); + + let mut loose = Vec::new(); + let mut total_declared = 0usize; + let mut total_ops = 0usize; + + for case in corpus::CASES { + let Ok(ast) = engine.compile(case.source) else { + continue; + }; + let program = Compiler::new().compile(&ast); + let Ok(high_water) = program.verify() else { + continue; + }; + + let declared: Vec = std::iter::once(program.main().max_stack()).chain(program.functions().iter().map(|f| f.chunk.max_stack())).collect(); + + total_declared += high_water.iter().map(|n| *n as usize).sum::(); + total_ops += program.code().len(); + + if declared != high_water { + loose.push(format!("\n {}: declares {declared:?}, uses {high_water:?}", case.name,)); + } + } + + println!("\n{total_declared} stack slots declared across {total_ops} bytes of code"); + + assert!(loose.is_empty(), "{} chunks declare a stack they do not use:{}", loose.len(), loose.join(""),); +} + +/// Residuals are the work left to do, so the count is the progress metric. +/// +/// Prints the whole census so a change in coverage is visible, and pins the +/// cases that should already be at zero. +#[test] +fn residual_census() { + let engine = corpus::engine(); + + let mut total_nodes = 0usize; + let mut at_zero = Vec::new(); + let mut remaining = Vec::new(); + let mut regressions = Vec::new(); + + // Cases the build removed are counted on neither side, or the completeness + // check below would read their absence as a corpus that stopped compiling. + let applicable = corpus::CASES.iter().filter(|case| applies_to_this_build(case.name)).count(); + + for case in corpus::CASES.iter().filter(|c| applies_to_this_build(c.name)) { + let Ok(ast) = engine.compile(case.source) else { + continue; + }; + let program = Compiler::new().compile(&ast); + let count = program.residual_count(); + let nodes = program.residual_nodes(); + total_nodes += nodes; + + if count == 0 { + at_zero.push(case.name); + } else { + remaining.push((case.name, nodes)); + if !MAY_FRAGMENT.contains(&case.name) && applies_to_this_build(case.name) { + regressions.push(format!("\n {} leaves {count}", case.name)); + } + } + } + + println!("\n{} of {applicable} scripts fully lowered, {total_nodes} AST nodes still in fragments", at_zero.len(),); + println!("\nfully lowered: {}", at_zero.join(", ")); + println!("\nremaining:"); + remaining.sort_by_key(|(_, count)| std::cmp::Reverse(*count)); + for (name, count) in &remaining { + println!(" {count:>3} {name}"); + } + + assert!( + regressions.is_empty(), + "{} scripts fragment that are not on `MAY_FRAGMENT`. Either the \ + construct regressed, or the case needs one of the four things the \ + escape hatch is for — say which, in the list:{}", + regressions.len(), + regressions.join(""), + ); + + // The other direction, which the check above cannot see: a corpus that + // stopped compiling at all would have nothing to fragment and would pass. + assert_eq!(at_zero.len(), applicable - MAY_FRAGMENT.len(), "some scripts did not compile, so they were counted as neither",); +} diff --git a/tests/grain/fixtures/follow.rhai b/tests/grain/fixtures/follow.rhai new file mode 100644 index 000000000..1e9d45480 --- /dev/null +++ b/tests/grain/fixtures/follow.rhai @@ -0,0 +1,296 @@ +fn pickn(ord, start, k) { + let out = [false, false, false]; + let i = 0; + while i < k && i < 3 { + out[ord[(start + i) % 3]] = true; + i += 1; + } + out +} + +fn order3(a, b, c) { + let ord = [0, 1, 2]; + let lev = [a, b, c]; + if lev[ord[1]] > lev[ord[0]] { let t = ord[0]; ord[0] = ord[1]; ord[1] = t; } + if lev[ord[2]] > lev[ord[1]] { let t = ord[1]; ord[1] = ord[2]; ord[2] = t; } + if lev[ord[1]] > lev[ord[0]] { let t = ord[0]; ord[0] = ord[1]; ord[1] = t; } + ord +} + +let STEP = 20; +let PEAK_DECAY = 0.999; +let PEAK_FLOOR = 20.0; +let FLUX_SMOOTH = 0.30; +let THR_SMOOTH = 0.02; +let THR_MULT = 1.8; +let THR_BIAS = 0.03; +let REFRACT = 110; +let BEAT_MIN = 380; +let BEAT_MAX = 680; +let BEAT_GAP = 6000; +let AGREE_PCT = 25; +let LOOK_BEATS = 8; +let DEAD_MS = 8000; +let PATTERN_MAX = 700; +let DIMMER_CH = 4; +let PAN_CH = 6; +let HOT_CH = 11; +let PHRASE_MIN = 4000; +let VOTE_SEQ = 3; +let COLOUR_DEAD = 0.30; +let WIDE_AT = 0.18; +let FULL_AT = 0.45; + +let DWELL = lamp_dwell_ms(); +if DWELL < 1 { DWELL = 1; } + +let pk_r = PEAK_FLOOR; +let pk_g = PEAK_FLOOR; +let pk_b = PEAK_FLOOR; +let pk_e = PEAK_FLOOR; +let e_prev = 0.0; +let flux = 0.0; +let thr = 0.05; +let lvl = 0.0; + +let last_onset = -99999; +let period = 500; +let anchor = 0; +let disagree = 0; +let last_seq = 0; + +let ord = [0, 1, 2]; +let pan_s = 0.0; +let pan_prev = 0.0; +let pan_dir = 0; +let pan_hi = 0.0; +let pan_lo = 0.0; +let pan_ref = 0.0; +let phrase_at = -99999; +let look_at = -99999; +let hot = false; +let gated = false; + +let beat_count = 0; +let last_slot = -1; +let scatter_start = 0; +let scatter_w = 1; + +let look = 0; +let look_until = 0; +let accent_until = -99999; + +let cur_r = false; +let cur_y = false; +let cur_g = false; +let last_change = -99999; + +let nudge = false; +let nudge_s = [false, false, false]; + +let t0 = millis(); +let next = t0; +let last_pkt = t0; +anchor = t0; + +loop { + let p = dmx_recv(STEP); + let now = millis(); + + if p.ok { + last_pkt = now; + + let dseq = p.seq - last_seq; + if dseq < 1 { dseq = 1; } + if dseq > 32 { dseq = 32; } + last_seq = p.seq; + + let r = p.ch[0].to_float(); + let g = p.ch[1].to_float(); + let b = p.ch[2].to_float(); + + pk_r *= PEAK_DECAY; + pk_g *= PEAK_DECAY; + pk_b *= PEAK_DECAY; + if pk_r < PEAK_FLOOR { pk_r = PEAK_FLOOR; } + if pk_g < PEAK_FLOOR { pk_g = PEAK_FLOOR; } + if pk_b < PEAK_FLOOR { pk_b = PEAK_FLOOR; } + if r > pk_r { pk_r = r; } + if g > pk_g { pk_g = g; } + if b > pk_b { pk_b = b; } + + let lr = r / pk_r; + let lb = b / pk_b; + let lg = g / pk_g; + let cmax = lr; + if lb > cmax { cmax = lb; } + if lg > cmax { cmax = lg; } + if cmax < COLOUR_DEAD { + let s0 = beat_count % 3; + ord = [s0, (s0 + 1) % 3, (s0 + 2) % 3]; + } else { + ord = order3(lr, lb, lg); + } + + let raw = r; + if g > raw { raw = g; } + if b > raw { raw = b; } + if p.ch.len() > DIMMER_CH { raw = p.ch[DIMMER_CH].to_float(); } + pk_e *= PEAK_DECAY; + if pk_e < PEAK_FLOOR { pk_e = PEAK_FLOOR; } + if raw > pk_e { pk_e = raw; } + let e = raw / pk_e; + + if p.ch.len() > PAN_CH { + pan_s += (p.ch[PAN_CH].to_float() - pan_s) * 0.08; + if pan_s > pan_hi { pan_hi = pan_s; } else { pan_hi += (pan_s - pan_hi) * 0.005; } + if pan_s < pan_lo { pan_lo = pan_s; } else { pan_lo += (pan_s - pan_lo) * 0.005; } + let dir = pan_dir; + if pan_s > pan_prev + 0.4 { dir = 1; } + else if pan_s < pan_prev - 0.4 { dir = -1; } + if dir != pan_dir && pan_dir != 0 && now - phrase_at > PHRASE_MIN { + let span = pan_hi - pan_lo; + let trav = pan_s - pan_ref; + if trav < 0.0 { trav = -trav; } + if span > 8.0 && trav > span * 0.35 { + phrase_at = now; + pan_ref = pan_s; + } + } + pan_dir = dir; + pan_prev = pan_s; + } + + if p.ch.len() > HOT_CH + 2 { + gated = true; + let h = p.ch[HOT_CH]; + if p.ch[HOT_CH + 1] > h { h = p.ch[HOT_CH + 1]; } + if p.ch[HOT_CH + 2] > h { h = p.ch[HOT_CH + 2]; } + hot = h > 24; + } + + let d = (e - e_prev) / dseq.to_float(); + if d < 0.0 { d = 0.0; } + e_prev = e; + flux += (d - flux) * FLUX_SMOOTH; + lvl += (e - lvl) * 0.06; + + if flux > thr * THR_MULT + THR_BIAS && now - last_onset >= REFRACT + && dseq <= VOTE_SEQ { + let iv = now - last_onset; + last_onset = now; + accent_until = now + DWELL; + + if iv <= BEAT_GAP { + while iv < BEAT_MIN { iv *= 2; } + while iv > BEAT_MAX { iv /= 2; } + if iv >= BEAT_MIN { + let err = iv - period; + if err < 0 { err = -err; } + if err * 100 <= period * AGREE_PCT { + period = (period * 4 + iv) / 5; + let ph = (now - anchor) % period; + if ph * 2 > period { ph -= period; } + let mag = ph; + if mag < 0 { mag = -mag; } + if mag * 4 > period { anchor = now; } else { anchor += ph / 4; } + disagree = 0; + } else { + disagree += 1; + if disagree >= 3 { + period = iv; + anchor = now; + disagree = 0; + } + } + } + } + } + thr += (flux - thr) * THR_SMOOTH; + } + + let pd = period; + while pd > PATTERN_MAX { pd /= 2; } + while pd < DWELL * 4 { pd *= 2; } + + let since = now - anchor; + if since < 0 { since = 0; } + let frac = (since % pd) * 1000 / pd; + let quarter = frac / 250; + let half = frac / 500; + + let live = now - last_pkt < DEAD_MS; + let wide_at = WIDE_AT; + let full_at = FULL_AT; + if gated && hot { + wide_at *= 0.6; + full_at *= 0.7; + } + let w = 1; + if lvl > wide_at { w = 2; } + if lvl > full_at { w = 3; } + if live && w < 2 { w = 2; } + let rest = w - 1; + if rest < 1 { rest = 1; } + + if quarter != last_slot { + if quarter == 0 { beat_count += 1; } + nudge = false; + if quarter % 2 == 0 { + scatter_start = rand_int(0, 2); + scatter_w = rand_int(1, w); + } + last_slot = quarter; + } + + if beat_count >= look_until || phrase_at > look_at { + look_at = phrase_at; + look_until = beat_count + LOOK_BEATS; + look = rand_int(0, 5); + } + + let s = [false, false, false]; + + if !live { + s = pickn(ord, beat_count, 2); + } else if now < accent_until { + s = pickn(ord, 0, 3); + } else if look == 0 { + if quarter < 2 { s = pickn(ord, 0, w); } else { s = pickn(ord, 0, rest); } + } else if look == 1 { + s = pickn(ord, beat_count * 2 + half, 1); + } else if look == 2 { + if quarter == 0 || quarter == 2 { s = pickn(ord, 0, w); } else { s = pickn(ord, half, rest); } + } else if look == 3 { + if quarter >= 2 { s = pickn(ord, half, w); } else { s = pickn(ord, beat_count, rest); } + } else if look == 4 { + let k = quarter + 1; + if k > 3 { k = 1; } + if k > w + 1 { k = w + 1; } + s = pickn(ord, 0, k); + } else { + s = pickn(ord, scatter_start, scatter_w); + } + + if !nudge && now - last_change >= pd / 2 { + nudge = true; + if cur_r || cur_y || cur_g { + nudge_s = [false, false, false]; + } else { + let k = w; + if k < 1 { k = 1; } + nudge_s = pickn(ord, beat_count, k); + } + } + if nudge { s = nudge_s; } + + if (s[0] != cur_r || s[1] != cur_y || s[2] != cur_g) && now - last_change >= DWELL { + set_lights(s[0], s[1], s[2]); + cur_r = s[0]; cur_y = s[1]; cur_g = s[2]; + last_change = now; + } + + next += STEP; + if next < now { next = now; } + sleep_until(next); +} diff --git a/tests/grain/fixtures/golden.rgrn b/tests/grain/fixtures/golden.rgrn new file mode 100644 index 000000000..b54feead3 Binary files /dev/null and b/tests/grain/fixtures/golden.rgrn differ diff --git a/tests/grain/fixtures/golden.rhai b/tests/grain/fixtures/golden.rhai new file mode 100644 index 000000000..6ad31fbb4 --- /dev/null +++ b/tests/grain/fixtures/golden.rhai @@ -0,0 +1,92 @@ +// The script behind `tests/fixtures/golden.rgrn`. See +// `an_artifact_written_by_an_older_build_still_runs` in `tests/format.rs`. +// +// Deliberately wide rather than deliberately short: every encoder branch this +// reaches is a branch the golden artifact pins against silent drift. Editing it +// means regenerating the artifact, which is the point — the pair only proves +// anything while the artifact is older than the reader. +// +// No `switch`. A switch table carries hashes from rhai's per-process hasher +// seed, so an artifact holding one is refused by any process that did not set +// the same seed (`format/write.rs:237`). That is right for a real artifact and +// useless for a fixture, which has to load anywhere. + +fn tally(items, bias) { + let total = bias; + for item in items { + total += item; + } + total +} + +const STRIDE = 2; +let counts = [1, 2, 3]; +let label = "measure"; +let ratio = 2.5; +let flags = #{ on: true, off: false, mark: 'x' }; +// Not all-constant, so the optimizer leaves it to be built at run time. +let computed = [STRIDE, counts.len(), ratio.to_int()]; + +// A chain rooted at a slot, one on a temporary, and one on the name the caller +// supplied — the three `Root` variants. +counts.push(tally(counts, 10)); +let widest = [9, 8].len(); +supplied.push(counts.len()); + +// The function-call spelling of the same method, which takes its first +// argument by reference — once over a slot and once over the caller's name, +// where the receiver is resolved after the other argument and moved back under +// it. +push(counts, ratio.to_int()); +push(supplied, STRIDE); + +// A closure, which shares what it captures and curries it onto a pointer. Made +// and spent inside a block, so the scope this is compared on does not end up +// holding a pointer — the two sides render one differently on purpose, which +// `a_closure_pointer_is_late_bound` in `tests/scope.rs` pins. +let captured = 0; +{ + let capture = || supplied.len() + counts.len(); + captured = capture.call(); +} + +let doubled = 0; +let i = 0; +while i < counts.len() { + doubled += counts[i]; + i += 1; +} + +let caught = (); +try { + if doubled > 0 || flags.off { + throw "thrown"; + } + caught = counts[99]; +} catch (err) { + caught = err; +} + +let squares = []; +for n in 0..8 { + if n > 3 { + break; + } + squares.push(n * STRIDE); +} + +#{ + total: doubled, + text: `${label} of ${counts.len()}`, + ratio: ratio * 2.0, + flag: flags.on, + mark: flags.mark, + supplied: supplied.len(), + widest: widest, + caught: caught, + captured: captured, + computed: computed, + squares: squares, + head: label[0..2], + tail: label[1..=3], +} diff --git a/tests/grain/format.rs b/tests/grain/format.rs new file mode 100644 index 000000000..3036a9b10 --- /dev/null +++ b/tests/grain/format.rs @@ -0,0 +1,703 @@ +//! An artifact must mean what the program it came from meant, and nothing a +//! wire can hand it may take the process down. +//! +//! Two separate claims, and they need separate tests. The first is a round +//! trip: compile, write, read back, run, and get what running the original +//! got — including the scope left behind and the exact error position. The +//! second is that `read` is total over arbitrary bytes: every truncation and +//! every single-byte corruption of a valid artifact either loads or fails, and +//! never panics. + +use super::corpus; + +use rhai::grain::format::{ReadError, WriteError}; +use rhai::grain::{Compiler, Program, Vm}; +use rhai::{Dynamic, Engine, Scope, INT}; + +/// What a run produced, in a form two runs can be compared on. +#[derive(Debug, PartialEq, Eq)] +struct Outcome { + result: Result, + scope: Vec<(String, String)>, +} + +/// A finished run, reduced to what two of them can be compared on. +fn snapshot(scope: &Scope, result: Result>) -> Outcome { + Outcome { + result: result.map(|value| format!("{value:?}")).map_err(|err| format!("{err:?}")), + scope: scope.iter_raw().map(|(name, _, value)| (name.to_string(), format!("{value:?}"))).collect(), + } +} + +/// Taken by value, because a program that hands pointers to natives has to be +/// owned to be run at all — and whether this one does is read back off the +/// bytes, which is the property that makes an artifact self-describing. +fn run(engine: &Engine, program: Program) -> Outcome { + let mut scope = Scope::new(); + let result = if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(engine).eval_with_callbacks(&mut scope, &program) + } else { + Vm::new(engine).eval_with_scope(&mut scope, &program) + }; + + snapshot(&scope, result) +} + +fn run_stock(engine: &Engine, source: &str) -> Outcome { + let mut scope = Scope::new(); + let ast = engine.compile(source).expect("corpus scripts parse"); + let result = engine.eval_ast_with_scope::(&mut scope, &ast); + + snapshot(&scope, result) +} + +/// Every corpus script that can be written, with its bytes. +fn writable(engine: &Engine) -> Vec<(&'static str, &'static str, Vec)> { + corpus::CASES + .iter() + .filter(|case| corpus::applies_to_this_build(case.name)) + .filter_map(|case| { + let ast = engine.compile(case.source).ok()?; + let bytes = Compiler::new().compile(&ast).write().ok()?; + Some((case.name, case.source, bytes)) + }) + .collect() +} + +/// The claim the format exists to support: bytes in one process mean the same +/// program in another. +#[test] +fn an_artifact_runs_as_the_program_it_came_from() { + let engine = corpus::engine(); + let mut failures = Vec::new(); + + for (name, source, bytes) in writable(&engine) { + let reloaded = match Program::read(&bytes) { + Ok(program) => program, + Err(err) => { + failures.push(format!("\n {name}: wrote but could not read back: {err}")); + continue; + } + }; + + let expected = run_stock(&engine, source); + let actual = run(&engine, reloaded); + + if expected != actual { + failures.push(format!("\n {name}: {source}\n rhai: {expected:?}\n artifact: {actual:?}")); + } + } + + assert!(failures.is_empty(), "{} artifacts do not mean what they came from:{}", failures.len(), failures.join(""),); +} + +/// A round trip over an empty set passes trivially, so pin the size of the set +/// and pin that the things it should contain are in it. +#[test] +fn the_round_trip_covers_something_worth_covering() { + let engine = corpus::engine(); + let written = writable(&engine); + let names: Vec<_> = written.iter().map(|(name, ..)| *name).collect(); + + assert!(written.len() >= 20, "only {} corpus scripts are writable, which is too few to prove anything: {names:?}", written.len(),); + + // One per construct the encoder has a branch for, so a branch that stops + // working names itself. + for required in [ + "int_arithmetic", // Call with an operator token + // A float constant, whose width the ABI pins — and which `no_float` + // removes from the language, so there is no branch left to cover. + #[cfg(not(feature = "no_float"))] + "float_arithmetic", + "shadowing_nested", // DeclareLocal and UnwindTo + "while_loop", // jumps, Tick, AssignLocal with an op + "loop_break_value", // backpatched jumps + // A position that has to survive. `unchecked` turns the failure it + // rests on into a panic in rhai, so the case is not run at all there. + #[cfg(not(feature = "unchecked"))] + "error_divide_by_zero", + "switch_range", // a switch table, and the hasher probe with it + "switch_guard", // and one whose arms are a chain rather than a target + "string_slice_read", // a range constant, which is a host type in `Dynamic` + "string_slice_inclusive", // and the other range tag + "index_assign_array", // a chain rooted at a slot, and its name + "temp_root_array_method", // and one rooted on the operand stack instead + ] { + assert!(names.contains(&required), "`{required}` no longer writes, so the encoder branch it covers is untested",); + } +} + +/// Where the golden pair lives. The source is checked in beside the artifact so +/// a regeneration is a visible two-file change. +const GOLDEN_SOURCE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/grain/fixtures/golden.rhai"); +const GOLDEN_ARTIFACT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/grain/fixtures/golden.rgrn"); + +/// The caller state `golden.rhai` expects. Part of the fixture, so it lives +/// with it rather than being invented at each use. +fn golden_scope() -> Scope<'static> { + let mut scope = Scope::new(); + scope.push("supplied", vec![Dynamic::from(7 as INT)]); + scope +} + +/// The one claim every other test in this file is blind to: that an artifact +/// written *earlier* still means the same thing. +/// +/// Named for `golden` so the regeneration command below selects it and nothing +/// else. +/// +/// Every other artifact here is produced by the current writer in the same +/// process, so a writer and reader that drift together agree with each other +/// perfectly and nothing notices. The device is the case that matters — bytes +/// built by one version of this crate and run by another — and a checked-in +/// artifact is the only way to have one side of that be genuinely old. +/// +/// Failing this is not automatically a bug. It means the encoding moved, and +/// the question it asks is whether that was deliberate. If it was, regenerate: +/// +/// ```text +/// REGENERATE_GOLDEN=1 cargo test --features grain --test grain golden +/// ``` +/// +/// and bump `VERSION` if an older reader would *misread* the new bytes rather +/// than reject them — the rule is at `src/format/mod.rs:56`. +#[test] +fn a_golden_artifact_written_by_an_older_build_still_runs() { + // The fixture is one build's bytes and its source is that build's source, + // which uses floats. `no_float` cannot parse it, so there is nothing here + // to check — the same reason the ABI guard skips below, reached earlier. + #[cfg(feature = "no_float")] + { + println!("skipped: the golden source uses floats, which this build has no syntax for"); + return; + } + + #[cfg(not(feature = "no_float"))] + { + let engine = corpus::engine(); + let source = std::fs::read_to_string(GOLDEN_SOURCE).expect("the golden source is checked in"); + let ast = engine.compile(&source).expect("the golden source must parse"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "the golden source must lower whole, or the artifact covers less than it claims: {:?}", program.first_unsupported(),); + + if std::env::var_os("REGENERATE_GOLDEN").is_some() { + let bytes = program.write().expect("the golden source must be writable"); + std::fs::write(GOLDEN_ARTIFACT, &bytes).expect("must write the artifact"); + println!("wrote {} bytes to {GOLDEN_ARTIFACT}", bytes.len()); + return; + } + + let bytes = std::fs::read(GOLDEN_ARTIFACT).expect("the golden artifact is checked in"); + let loaded = match Program::read(&bytes) { + Ok(loaded) => loaded, + // The header records the ABI the fixture was written under, and a build + // with different numeric widths or restriction flags refuses it *by + // design* — that refusal is what `abi.rs` is for. The fixture is one + // build's bytes, so it can only be checked on that build; anywhere else + // this would be testing the ABI guard rather than the encoding. + Err(err) if format!("{err}").contains("written with") => { + println!("skipped: the golden fixture is a default-build artifact ({err})"); + return; + } + Err(err) => panic!( + "the golden artifact no longer loads: {err}\n\ + The format moved. If that was deliberate, regenerate the fixture with \ + `REGENERATE_GOLDEN=1 cargo test --features grain --test grain golden`.", + ), + }; + + // A fixture only pins what it contains, and narrowing one while editing the + // source is easy and silent. These are read off the *artifact*, so they say + // what the encoder branch coverage actually is rather than what the source + // looks like it should give. + let kinds: std::collections::BTreeSet = rhai::grain::bytecode::disassemble(loaded.code()) + .map(|(_, op)| format!("{op:?}").split(['(', ' ', '{']).next().unwrap_or_default().to_string()) + .collect(); + for required in [ + "Chain", // a chain record, with all three of its roots + "CallRef", // and both by-reference call forms + "Rotate", // which only a named receiver needs + "LoadNamed", // the caller's variable, flat + "LoadSharedNamed", // and as the cell a capture binds + "MakeClosure", // a function pointer to a compiled chunk + "Curry", // with what it captured bound onto it + "MakeArray", // a literal the optimizer could not fold + "MakeMap", // and its template-plus-pairs cousin + "CheckSize", // the per-element size check beside it + "PushHandler", // a handler region, whose catch variable is pooled + "Throw", // + "IterNext", // an iterator, and the two-edged instruction + "InterpolateAppend", // a string built a segment at a time + ] { + assert!( + kinds.contains(required), + "the golden no longer contains `{required}`, so its encoder branch \ + is unpinned again — put it back or say why it went", + ); + } + assert!( + kinds.len() >= 35, + "the golden covers only {} instruction kinds, which is narrower than it \ + was written to be: {kinds:?}", + kinds.len(), + ); + + let walked = { + let mut scope = golden_scope(); + let result = engine.eval_ast_with_scope::(&mut scope, &ast); + snapshot(&scope, result) + }; + let ran = { + let mut scope = golden_scope(); + // Whether the program can hand a pointer to a native is read back off + // the bytes, so how it must be run is part of what is being checked. + let result = if loaded.makes_fn_pointers() { + let loaded = loaded.into_shared(); + Vm::new(&engine).eval_with_callbacks(&mut scope, &loaded) + } else { + Vm::new(&engine).eval_with_scope(&mut scope, &loaded) + }; + snapshot(&scope, result) + }; + + assert_eq!( + ran, walked, + "the golden artifact no longer means what its source means.\n\ + The format moved without the reader noticing, which is the failure this \ + fixture exists to catch. If the change was deliberate, regenerate with \ + `REGENERATE_GOLDEN=1 cargo test --features grain --test grain golden`.", + ); + assert!(ran.result.is_ok(), "the golden must produce a value, not an error: {ran:?}",); + } +} + +/// A chain rooted at a caller's variable, which the corpus cannot cover. +/// +/// Every case in `writable` runs from an empty scope, so the name a +/// [`Root::Named`] carries — and the position beside it, which is what an +/// `ErrorVariableNotFound` is reported against — has no encoder coverage there. +#[test] +fn a_chain_rooted_at_a_name_survives_the_round_trip() { + let engine = corpus::engine(); + let source = "host.push(2); host[9]"; + + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "the chain must lower"); + let bytes = program.write().expect("must be writable"); + let reloaded = Program::read(&bytes).expect("what we wrote must read back"); + + let seed = |scope: &mut Scope| { + scope.push("host", vec![Dynamic::from(1 as INT)]); + }; + + let mut walked = Scope::new(); + seed(&mut walked); + let expected = { + let out = engine.eval_ast_with_scope::(&mut walked, &ast); + snapshot(&walked, out) + }; + + let mut loaded = Scope::new(); + seed(&mut loaded); + let actual = { + let out = Vm::new(&engine).eval_with_scope(&mut loaded, &reloaded); + snapshot(&loaded, out) + }; + + // The mutation lands, and the out-of-bounds index is still blamed on the + // step rather than on the chain — so both the name and its position came + // back intact. + assert_eq!(actual, expected); + assert!(actual.result.is_err(), "the index must still be refused"); +} + +/// Fragments are the allocation the format exists to remove, so writing one +/// would defeat the point. +/// +/// The refusal has to name the construct and where it is. A caller deciding +/// whether to ship source instead cannot act on "27 fragments"; it can act on +/// "for at line 1". +#[test] +fn refusing_to_write_names_the_construct_responsible() { + let engine = corpus::engine(); + + for (source, expected) in [ + ("let x = 1; eval(\"x\")", "an unlowered expression"), + // `?.` short-circuits on unit rather than stepping, so it is not a + // chain this compiler can express whatever its root is. + ("let x = 1; x?.y", "an unlowered expression"), + ] { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert!(program.residual_count() > 0, "{source:?} must still fragment, or this test has gone stale",); + + let Err(err @ WriteError::HasResiduals { construct, pos, .. }) = program.write() else { + panic!("{source:?} must refuse to write"); + }; + assert_eq!(construct, expected, "for {source:?}"); + // Under `no_position` there is no "where" to say, and the naming half + // above is the part that still means something. + #[cfg(not(feature = "no_position"))] + assert!(!pos.is_none(), "the refusal must say where: {err}"); + let _ = pos; + assert!(err.to_string().contains(expected), "{err}"); + } +} + +/// A script function is a chunk like any other, so it crosses the wire with +/// the rest of the program. +#[test] +fn script_functions_survive_the_round_trip() { + let engine = corpus::engine(); + + for source in [ + "fn add(a, b) { a + b } add(2, 3)", + "fn fib(n) { if n < 2 { n } else { fib(n - 1) + fib(n - 2) } } fib(6)", + "fn first() { 1 } fn second(x) { first() + x } second(4)", + // Failing inside a function has to keep rhai's wrapping and position. + // `unchecked` turns this into a panic in rhai's own built-in rather + // than an error, so there is nothing left here to compare. + #[cfg(not(feature = "unchecked"))] + "fn bad(x) { x / 0 } bad(1)", + ] { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert!(!program.functions().is_empty(), "{source:?} must compile its functions, not leave them to the walker",); + + let bytes = program.write().expect("must be writable"); + let reloaded = Program::read(&bytes).expect("must load"); + + assert_eq!(run(&engine, reloaded), run_stock(&engine, source), "{source:?} does not mean the same after a round trip",); + } +} + +/// A function the compiler cannot lower stays rhai's, and a program that still +/// depends on rhai's copy cannot be written — silently dropping it would +/// produce an artifact that loads and then cannot find its own function. +#[test] +#[cfg(not(feature = "no_module"))] +fn a_function_the_compiler_cannot_lower_refuses_to_write() { + let engine = corpus::engine(); + // `import` declares into the caller's scope, which the slot model cannot + // account for. `this` used to be the example here, and is not one any more. + let ast = engine.compile(r#"fn m() { import "x" as y; 1 } m()"#).expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert!(program.functions().is_empty(), "a body the slot model cannot account for must not become a chunk",); + assert!(matches!(program.write(), Err(WriteError::HasScriptFunctions | WriteError::HasResiduals { .. }),), "got {:?}", program.write(),); +} + +/// The counterpart, and the milestone: a body that uses `this` is a chunk now, +/// so a program full of them is an artifact rather than a tree. +#[test] +fn a_body_using_this_is_compiled_and_writable() { + let engine = corpus::engine(); + let ast = engine.compile("fn bump(n) { this += n; this } let x = 21; x.bump(21); x").expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert_eq!(program.residual_count(), 0, "{:?}", program.first_unsupported()); + assert!(!program.functions().is_empty(), "a body using `this` must become a chunk"); + assert!(program.write().is_ok(), "got {:?}", program.write()); +} + +/// A program with one of everything the corruption tests need to reach: a +/// float constant, a loop, and a `switch` — whose table is the one part of an +/// artifact holding jump targets that are not in the code, and so the one the +/// verifier would most plausibly forget to check. +/// +/// Both by-reference call forms are here too. Their argument count includes a +/// receiver that is not on the operand stack, so a corrupted one is read +/// against a different depth than any other call's. +fn sample(engine: &Engine) -> Vec { + // A float in the constant pool is part of what this covers, and `no_float` + // has no float to put there. The rest of the shape — a loop, an array, a + // caller variable, an indexed write, a switch — is the same either way. + #[cfg(not(feature = "no_float"))] + const SECOND: &str = "2.5"; + #[cfg(feature = "no_float")] + const SECOND: &str = "2"; + + let source = format!( + "let a = 1; let b = {SECOND}; while a < 10 {{ a += 1 }} \ + let c = [a]; push(c, b); push(caller_supplied, a); \ + caller_supplied[0] = a; \ + switch a {{ 1 => \"one\", 0..=20 => \"some\", _ => \"many\" }}" + ); + let ast = engine.compile(&source).expect("must compile"); + Compiler::new().compile(&ast).write().expect("the sample must be writable") +} + +#[test] +fn something_that_is_not_an_artifact_is_refused_at_the_first_bytes() { + assert_eq!(Program::read(b"").unwrap_err(), ReadError::Truncated); + assert_eq!(Program::read(b"not an artifact at all").unwrap_err(), ReadError::BadMagic,); +} + +#[test] +fn a_future_format_version_is_refused_rather_than_guessed_at() { + let engine = corpus::engine(); + let mut bytes = sample(&engine); + bytes[4] = 0xff; + bytes[5] = 0xff; + + assert!(matches!(Program::read(&bytes).unwrap_err(), ReadError::UnsupportedVersion { found: 0xffff, .. },)); +} + +/// The fingerprint is the difference between a clean failure and integers +/// decoded as the wrong type, so the error must name the flag. +#[test] +fn a_different_value_representation_is_refused_by_name() { + let engine = corpus::engine(); + + let mut narrow = sample(&engine); + // Halved rather than named: `only_i32` already makes 4 the host's own + // width, and an artifact agreeing with the host is not refused at all. + let half = narrow[6] / 2; // INT width + narrow[6] = half; + let message = Program::read(&narrow).unwrap_err().to_string(); + assert!(message.contains("INT") && message.contains(&half.to_string()), "the message must name the width: {message}",); + + let mut restricted = sample(&engine); + restricted[8] ^= 0b100; // the `no_index` bit + let message = Program::read(&restricted).unwrap_err().to_string(); + assert!(message.contains("no_index"), "the message must name the flag: {message}",); +} + +/// A `switch` carries hashes rhai's parser computed, and rhai seeds its hasher +/// per process unless the host says otherwise. Two processes that disagree +/// would load each other's artifacts perfectly and then send every subject to +/// the default — a wrong answer rather than a failure, which is the worst kind. +/// +/// The probe is what turns it into a failure, so this checks the failure +/// happens and that the message says what to do about it. +#[test] +fn a_switch_hashed_by_a_different_seed_is_refused() { + let engine = corpus::engine(); + let bytes = sample(&engine); + + // The probe is the only place the artifact repeats this value, and finding + // it that way means the test does not have to know the layout. + let probe = rhai::grain::bytecode::probe().to_le_bytes(); + let at = bytes.windows(probe.len()).position(|window| window == probe).expect("an artifact with a switch in it carries a probe"); + + let mut corrupt = bytes.clone(); + corrupt[at] ^= 1; + + let err = Program::read(&corrupt).expect_err("a foreign hasher must be refused"); + assert!(matches!(err, ReadError::HashSeedMismatch { .. }), "got {err:?}",); + assert!(err.to_string().contains("set_hashing_seed"), "the message must say how to fix it: {err}",); + + // And the uncorrupted one still loads, so the check is not simply always + // failing. + assert!(Program::read(&bytes).is_ok()); +} + +/// An artifact arrives over a link, so every prefix of one is a thing that can +/// actually turn up. None may load, and none may panic. +#[test] +fn every_truncation_fails_cleanly() { + let engine = corpus::engine(); + let bytes = sample(&engine); + + for cut in 0..bytes.len() { + assert!(Program::read(&bytes[..cut]).is_err(), "a {cut}-byte prefix of a {}-byte artifact loaded", bytes.len(),); + } + + assert!(Program::read(&bytes).is_ok(), "the whole thing must load"); +} + +/// Trailing bytes mean the file is not what it says it is, even though every +/// field parsed. Accepting them would let a valid artifact carry a payload. +#[test] +fn trailing_bytes_are_refused() { + let engine = corpus::engine(); + let mut bytes = sample(&engine); + bytes.push(0); + + assert_eq!(Program::read(&bytes).unwrap_err(), ReadError::TrailingBytes { count: 1 },); +} + +/// The safety claim in one test: a corrupted artifact is a `Result`, never a +/// panic and never a chunk the VM will touch. +/// +/// Flipping each bit of each byte is exhaustive over single-bit corruption, +/// which is what a bad link produces. Whatever survives has been through the +/// verifier, so it is safe to run — and running it here is what proves the +/// verifier is actually on the load path. +/// +/// **Verification is not termination.** A flipped jump target that still lands +/// inside the chunk is a structurally valid infinite loop, and this test hung +/// until it ran under a budget. That is not a gap to close — no loader can +/// decide halting — it is the reason `Op::Tick` sits on every back edge and the +/// reason the patch exposes `track_operation`. A host running untrusted +/// bytecode must set `max_operations`, exactly as it must for untrusted source. +#[test] +// A corrupted chunk can loop, and `max_operations` is what stops it. Without +// limits this hangs rather than fails, which is worse than not running. +#[cfg(not(feature = "unchecked"))] +fn no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk() { + let writer = Engine::new(); + let bytes = sample(&writer); + + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let mut loaded = 0usize; + + for index in 0..bytes.len() { + for bit in 0..8 { + let mut corrupt = bytes.clone(); + corrupt[index] ^= 1 << bit; + + if let Ok(program) = Program::read(&corrupt) { + loaded += 1; + program.verify().expect("read must not return a chunk that fails verification"); + // The result is free to be anything; not crashing is the claim. + let _ = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program); + } + } + } + + // Most flips land in a length, a tag or the fingerprint and are rejected. + // Some land in a constant's value and legitimately still load; that is the + // case worth having run above. + println!("{loaded} of {} single-bit corruptions still loaded", bytes.len() * 8,); +} + +/// The whole point of the split, end to end. +/// +/// The device is sent a stripped artifact and knows nothing about the source. +/// It fails, and all it can say is which instruction. The host kept the table, +/// and turns that back into the position rhai itself would have reported. +/// There is no table to strip under `no_position`, so nothing to resolve; and +/// the failure it turns on is a division by zero, which `unchecked` makes a +/// panic in rhai rather than an error. +#[test] +#[cfg(not(any(feature = "no_position", feature = "unchecked")))] +fn a_stripped_program_reports_an_address_the_host_can_resolve() { + let engine = corpus::engine(); + let source = "let a = 1;\nlet b = 0;\na / b"; + + // Host: compile, split. + let ast = engine.compile(source).expect("must compile"); + let full = Compiler::new().compile(&ast); + let expected = run_stock(&engine, source); + let (shipped, table) = full.write_stripped().expect("must be writable"); + + // Device: run bytes, with no table and no source. + let device = Program::read(&shipped).expect("the device must load it"); + assert!(device.positions().is_stripped(), "a stripped artifact must not carry positions",); + + let mut vm = Vm::new(&engine); + let error = vm.eval_with_scope(&mut Scope::new(), &device).expect_err("dividing by zero must fail"); + let address = vm.fault_pc().expect("a failed run must name an instruction"); + + // Host: resolve what came back. + let site = rhai::grain::pos::resolve(&table, address as u32).expect("the failing instruction must have a recorded site"); + + assert_eq!((site.line, site.column), (3, 3), "the division is at line 3, column 3 of {source:?}",); + + // And the same program with its table attached says so itself, exactly as + // rhai does — which is what makes the resolved site trustworthy. + let mut reattached = Program::read(&shipped).unwrap(); + reattached.attach_positions(&table).expect("its own table must attach"); + assert_eq!(run(&engine, reattached), expected); + + // The stripped run is the same failure, minus the position. + assert!(error.position().is_none(), "a stripped program has no position to report, got {:?}", error.position(),); +} + +/// Attaching another program's table would misreport every error rather than +/// reporting none, which is strictly worse than having no table. +#[test] +#[cfg(not(feature = "no_position"))] +fn a_table_from_a_different_program_is_refused() { + let engine = corpus::engine(); + + let short = Compiler::new().compile(&engine.compile("1 + 1").unwrap()); + let long = Compiler::new().compile(&engine.compile("let a = 1; while a < 9 { a += 1 } a").unwrap()); + + let (_, long_table) = long.write_stripped().expect("must be writable"); + let (short_bytes, _) = short.write_stripped().expect("must be writable"); + + let mut program = Program::read(&short_bytes).unwrap(); + assert!(program.attach_positions(&long_table).is_err(), "a table naming instructions this chunk does not have must be refused",); +} + +/// A stripped artifact that arrives with a table still in it is a contradiction +/// the reader should not paper over. +#[test] +#[cfg(not(feature = "no_position"))] +fn an_artifact_carrying_a_mismatched_table_does_not_load() { + let engine = corpus::engine(); + let bytes = sample(&engine); + let program = Program::read(&bytes).expect("the sample must load"); + + assert!(!program.positions().is_stripped(), "`write` keeps the table, so this one must have positions",); +} + +/// What the split costs, and what it saves. +#[test] +fn stripping_positions_shrinks_the_artifact() { + let engine = corpus::engine(); + + let mut with = 0usize; + let mut without = 0usize; + let mut tables = 0usize; + + for (name, _, full) in writable(&engine) { + let ast = engine.compile(corpus::CASES.iter().find(|c| c.name == name).unwrap().source); + let program = Compiler::new().compile(&ast.unwrap()); + let (stripped, table) = program.write_stripped().expect("must be writable"); + + with += full.len(); + without += stripped.len(); + tables += table.len(); + } + + println!( + "\n{with} bytes with positions -> {without} stripped ({:.0}% smaller), \ + {tables} bytes of table kept behind", + 100.0 * (with - without) as f64 / with as f64, + ); + + assert!(without < with, "stripping must actually remove something: {without} vs {with}",); +} + +/// The number this project exists to move: bytes retained per source byte, +/// against the 24 a rhai `AST` costs on device. +/// +/// This is the host-side artifact size, not device heap, which only a device +/// can report. What it establishes is the encoding's own density, which is the +/// part the format controls. +#[test] +fn artifact_size_census() { + let engine = corpus::engine(); + let written = writable(&engine); + + let mut source_bytes = 0usize; + let mut artifact_bytes = 0usize; + let mut rows: Vec<_> = written + .iter() + .map(|(name, source, bytes)| { + source_bytes += source.len(); + artifact_bytes += bytes.len(); + (*name, source.len(), bytes.len()) + }) + .collect(); + + rows.sort_by_key(|(_, _, artifact)| std::cmp::Reverse(*artifact)); + println!("\n{:>7} {:>7} script", "source", "bytes"); + for (name, source, artifact) in &rows { + println!("{source:>7} {artifact:>7} {name}"); + } + println!("\n{} scripts: {source_bytes} source bytes -> {artifact_bytes} artifact bytes ({:.2}x)", rows.len(), artifact_bytes as f64 / source_bytes as f64,); + + // Not a target, a tripwire. The plan is explicit that bytecode need not + // beat minified source on bytes — but an encoding several times larger + // than its input has a bug in it, not a tradeoff. + assert!(artifact_bytes < source_bytes * 3, "{artifact_bytes} artifact bytes for {source_bytes} of source is not an encoding",); +} diff --git a/tests/grain/fuzz.rs b/tests/grain/fuzz.rs new file mode 100644 index 000000000..0a7210c2e --- /dev/null +++ b/tests/grain/fuzz.rs @@ -0,0 +1,471 @@ +//! Randomised loading, on the assumption that an artifact is hostile. +//! +//! `Program::read` is the only place untrusted bytes enter, and what it hands +//! back is executed in place — so the claim it has to support is total: any +//! byte string either fails to load or loads into a chunk the VM can run +//! without panicking, looping forever, or reading outside itself. +//! +//! `no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk` in `format.rs` is the +//! exhaustive half, over one-bit corruption of one artifact. This is the wide +//! half: many mutations, several of them structural, over a corpus of real +//! artifacts. Seeded rather than random, so a failure is reproducible and CI +//! does not go intermittent — the seed is printed and any interesting input is +//! dumped as hex. +//! +//! It is not a substitute for `cargo fuzz` (see `fuzz/`), which explores with +//! coverage feedback. It is what runs on every `cargo test`. + +// Only the sources are wanted here; the names belong to the harnesses that +// report per-case results. +use super::corpus; + +use rhai::grain::{Compiler, Program, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +use super::corpus::generate::{Generator, Rng}; + +/// Ways an artifact can arrive wrong. Truncation and splicing matter as much +/// as corruption: a length field that disagrees with what follows is how a +/// loader is talked into reading past the end. +fn mutate(rng: &mut Rng, bytes: &[u8]) -> Vec { + let mut out = bytes.to_vec(); + if out.is_empty() { + return out; + } + + match rng.below(6) { + // Corrupt a run of bytes, which reaches multi-byte fields that + // single-bit flipping cannot. + 0 | 1 => { + let at = rng.below(out.len()); + let run = 1 + rng.below((out.len() - at).min(8)); + for byte in &mut out[at..at + run] { + *byte = rng.next() as u8; + } + } + // Cut it short. + 2 => out.truncate(rng.below(out.len())), + // Splice out a slice, so every following offset is wrong. + 3 => { + let at = rng.below(out.len()); + let run = 1 + rng.below((out.len() - at).min(16)); + out.drain(at..at + run); + } + // Insert junk, the same problem in the other direction. + 4 => { + let at = rng.below(out.len()); + let junk: Vec = (0..1 + rng.below(16)).map(|_| rng.next() as u8).collect(); + out.splice(at..at, junk); + } + // Set a byte to an edge value: the lengths and counts are varints, and + // 0xff runs are what make one disagree with reality. + _ => { + let at = rng.below(out.len()); + out[at] = [0x00, 0x01, 0x7f, 0x80, 0xff][rng.below(5)]; + } + } + out +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// The claim, over a lot of inputs. +/// +/// Running what survives is the point rather than a bonus: a loader that +/// accepts a chunk it should not have has done nothing visible until something +/// executes it. +#[test] +fn mutated_artifacts_load_or_fail_but_never_misbehave() { + let writer = corpus::engine(); + + // Every artifact the corpus can produce, so the mutations land on real + // structure — a hand-written sample would only ever exercise its own + // shape. + let seeds: Vec> = corpus::CASES + .iter() + .filter_map(|case| { + let ast = writer.compile(case.source).ok()?; + Compiler::new().compile(&ast).write().ok() + }) + .collect(); + + assert!(seeds.len() >= 50, "only {} artifacts to mutate, which is too few to prove anything", seeds.len(),); + + // A budget, because verification proves structure and not termination: a + // corrupted-but-in-range jump target is a valid infinite loop. + let mut engine = corpus::engine(); + engine.set_max_operations(10_000); + engine.set_max_string_size(4096); + engine.set_max_array_size(1024); + + const SEED: u64 = 0x5eed_1234_abcd_0001; + // Enough to be worth running on every `cargo test` and not so many that + // anyone is tempted to stop. Depth is `cargo fuzz`'s job — see `fuzz/`. + const ROUNDS: usize = 20; + + let mut rng = Rng::new(SEED); + let mut loaded = 0usize; + + for round in 0..ROUNDS { + for original in &seeds { + let corrupt = mutate(&mut rng, original); + + let Ok(program) = Program::read(&corrupt) else { + continue; + }; + loaded += 1; + + // Anything `read` returns must already have verified — that is the + // contract the VM's missing bounds checks rest on. + assert!( + program.verify().is_ok(), + "seed {SEED:#x} round {round}: read returned a chunk that does \ + not verify: {}", + hex(&corrupt), + ); + + // And must then run without taking the process down. The result is + // free to be anything at all. + let _ = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program); + } + } + + println!("{loaded} of {} mutations loaded and ran", ROUNDS * seeds.len(),); + assert!(loaded > 0, "no mutation survived, so nothing was actually executed",); +} + +/// Bytes that were never an artifact, which is the other way in. +#[test] +fn arbitrary_bytes_never_load_into_something_that_misbehaves() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let mut rng = Rng::new(0x5eed_0000_0000_0002); + + for _ in 0..20_000 { + let len = rng.below(64); + let mut bytes: Vec = (0..len).map(|_| rng.next() as u8).collect(); + + // Half of them get the magic, so the loader is reached rather than + // rejected at the first four bytes every time. + if rng.below(2) == 0 && bytes.len() >= 4 { + bytes[..4].copy_from_slice(b"RGRN"); + } + + if let Ok(program) = Program::read(&bytes) { + assert!(program.verify().is_ok(), "unverified: {}", hex(&bytes)); + let _ = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program); + } + } +} + +/// What a script produced, in a form two runs can be compared on. +#[derive(PartialEq, Eq)] +struct Outcome { + result: Result, + scope: Vec<(String, String)>, +} + +impl std::fmt::Debug for Outcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.result { + Ok(value) => write!(f, "{value} | scope {:?}", self.scope), + Err(err) => write!(f, "!{err} | scope {:?}", self.scope), + } + } +} + +fn snapshot(scope: &Scope, result: Result>) -> Outcome { + Outcome { + result: result.map(|value| format!("{value:?}")).map_err(|err| format!("{err:?}")), + scope: scope.iter_raw().map(|(name, _, value)| (name.to_string(), format!("{value:?}"))).collect(), + } +} + +/// Whether an outcome is one the two sides are not expected to reach together. +/// +/// Both enforce these limits; neither counts towards them in lockstep, because +/// a VM instruction is not a walker node and a callback boundary costs a +/// different number of call levels. A script that runs into one has stopped +/// saying anything about lowering, so it is dropped rather than compared — +/// and the drop rate is asserted, so this cannot quietly become the answer for +/// everything. +fn hit_a_limit(outcome: &Outcome) -> bool { + let Err(err) = &outcome.result else { + return false; + }; + ["ErrorTooManyOperations", "ErrorStackOverflow", "ErrorTooManyVariables"].iter().any(|limit| err.contains(limit)) +} + +/// Run something that may panic, without the panic reaching the console. +/// +/// The hook is swapped rather than left alone because a fuzzing run can trip +/// the same upstream panic thousands of times, and the backtrace for each one +/// is noise. Single-threaded, and restored immediately. +fn quietly(body: impl FnOnce() -> T) -> Option { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(body)).ok(); + std::panic::set_hook(hook); + out +} + +/// A rhai bug, not one of ours, found by `cargo fuzz run generated`. +/// +/// A `switch` as the last statement of a block makes rhai's optimizer delete a +/// `let` in that block and flatten what is left into the enclosing statement +/// list — while the reads of that local keep the scope index the parser gave +/// them. The index is counted back from the end of the scope, so it now names +/// whatever moved into its place. +/// +/// Two symptoms, and the quiet one is the dangerous one: +/// +/// * with something else at that index, rhai answers with **another variable's +/// value** and reports nothing at all; +/// * with nothing there, `scope.len() - index` underflows (`eval/expr.rs:131`) +/// — a panic in a debug build and a wild index in a release one. +/// +/// Both are three lines of ordinary rhai with none of this involved. We resolve +/// locals by name, so we say the variable is missing, which is the closest +/// thing to right available: there is no agreeing with an AST that refers to a +/// local it does not declare. +/// +/// Pinned because `generated_scripts_agree_with_the_walker` and both `cargo +/// fuzz` targets have to skip these, and every one of those skips should go the +/// day this test starts failing. +#[test] +fn rhai_drops_a_local_its_optimizer_still_refers_to() { + // The read lands on `a`, so rhai answers 1 where the script says 99. + const WRONG: &str = "let a = 1; { let b = 99; switch b { _ => b } }"; + // The same shape with nothing left at that index. + const PANIC: &str = "{ let b = 99; switch b { _ => b } }"; + + let engine = corpus::engine(); + let mut plain = corpus::engine(); + plain.set_optimization_level(rhai::OptimizationLevel::None); + + let ast = engine.compile(WRONG).expect("it parses"); + let value = engine.eval_ast::(&ast).expect("rhai runs it, which is the problem"); + assert_eq!( + value.as_int().ok(), + Some(1), + "rhai no longer reads the wrong variable — delete the optimizer skips \ + in the fuzzers and this test with them", + ); + + let ast = engine.compile(PANIC).expect("it parses"); + assert!( + quietly(|| engine.eval_ast::(&ast)).is_none(), + "rhai no longer underflows here — delete the walker skip in \ + `generated_scripts_agree_with_the_walker`", + ); + + // The same scripts with the optimizer out of the way, which is what says + // the fault is in the optimizer rather than in them. + for source in [WRONG, PANIC] { + let ast = plain.compile(source).expect("it parses either way"); + let value = quietly(|| plain.eval_ast::(&ast)) + .expect("with the optimizer off it should run") + .expect("and it should not fail"); + assert_eq!(value.as_int().ok(), Some(99), "{source:?}"); + } +} + +/// Whether the two sides agree once rhai's optimizer is out of the way. +/// +/// The optimizer is what makes a dropped local reachable, so agreeing without +/// it and disagreeing with it says the AST is at fault rather than the +/// lowering. Only ever asked about a divergence that already looks like one. +fn agree_unoptimised(source: &str) -> bool { + let mut plain = corpus::engine(); + plain.set_optimization_level(rhai::OptimizationLevel::None); + let Ok(ast) = plain.compile(source) else { + return false; + }; + + let mut walker_scope = Scope::new(); + let Some(walked) = quietly(|| plain.eval_ast_with_scope::(&mut walker_scope, &ast)) else { + return false; + }; + + let program = Compiler::new().compile(&ast); + let mut vm_scope = Scope::new(); + let ours = if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(&plain).eval_with_callbacks(&mut vm_scope, &program) + } else { + Vm::new(&plain).eval_with_scope(&mut vm_scope, &program) + }; + + snapshot(&walker_scope, walked) == snapshot(&vm_scope, ours) +} + +/// The claim the corpus makes, over scripts nobody wrote. +/// +/// `tests/differential.rs` pins the constructs someone thought of. This is the +/// same comparison — value, error variant, error position, and what was left in +/// the scope — over combinations of them that no one did. +#[test] +fn generated_scripts_agree_with_the_walker() { + // Budgets, not correctness limits: a generated script can be quadratic in + // a way no corpus case is, and there is no reason to wait for it. + let mut engine = corpus::engine(); + engine.set_max_operations(200_000); + engine.set_max_array_size(2048); + engine.set_max_string_size(8192); + // Pinned, because rhai's defaults for these are `debug_assertions`-gated — + // 32/16 and 8 in a debug build against 64/64 and 64 in a release one + // (`api/limits.rs:10-36`). Left alone, `cargo test` and + // `cargo test --release` parse different halves of the same seeded corpus + // and compare different scripts, which is not a thing a differential + // harness may do. The release numbers, because they admit more. + engine.set_max_expr_depths(64, 64); + engine.set_max_call_levels(64); + + const SEED: u64 = 0x5eed_9a11_0000_0001; + const SCRIPTS: usize = 4000; + + let mut parsed = 0usize; + let mut ran = 0usize; + let mut valued = 0usize; + let mut skipped = 0usize; + let mut walker_panics = 0usize; + let mut too_deep = 0usize; + let mut lost_a_local = 0usize; + let mut unparsed: Vec = Vec::new(); + let mut failures = Vec::new(); + + for n in 0..SCRIPTS { + // Seeded per script rather than from one long stream, so a failing + // script can be reproduced on its own by its index. + let source = Generator::new(SEED ^ n as u64).script(); + + let ast = match engine.compile(&source) { + Ok(ast) => ast, + Err(err) => { + if matches!(err.err_type(), rhai::ParseErrorType::ExprTooDeep) { + too_deep += 1; + } else { + unparsed.push(format!("\n {err}\n {source}")); + } + continue; + } + }; + parsed += 1; + + // The scope is snapshotted after the run, not during: what a script + // leaves behind is half of what is being compared. + // + // Guarded because the walker can panic on a script of its own accord — + // see `rhai_underflows_a_scope_index_under_its_own_optimizer`. There is + // nothing to compare against a side that did not finish, so those are + // counted and dropped rather than blamed on the VM. + let mut walker_scope = Scope::new(); + let Some(walked) = quietly(|| engine.eval_ast_with_scope::(&mut walker_scope, &ast)) else { + walker_panics += 1; + continue; + }; + let walked = snapshot(&walker_scope, walked); + + if hit_a_limit(&walked) { + skipped += 1; + continue; + } + ran += 1; + if walked.result.is_ok() { + valued += 1; + } + + let program = Compiler::new().compile(&ast); + let mut vm_scope = Scope::new(); + let ours = if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(&engine).eval_with_callbacks(&mut vm_scope, &program) + } else { + Vm::new(&engine).eval_with_scope(&mut vm_scope, &program) + }; + let ours = snapshot(&vm_scope, ours); + + if hit_a_limit(&ours) { + skipped += 1; + continue; + } + + if walked == ours { + continue; + } + // Not every disagreement is one to have: rhai's optimizer can delete a + // `let` whose variable is still read, and then there is no agreeing + // with it. See `rhai_drops_a_local_its_optimizer_still_refers_to`. + if format!("{ours:?}").contains("ErrorVariableNotFound") && agree_unoptimised(&source) { + lost_a_local += 1; + continue; + } + + if failures.len() < 5 { + failures.push(format!("\n=== script {n} (seed {:#x}) ===\n {source}\n rhai: {walked:?}\n vm: {ours:?}", SEED ^ n as u64,)); + } + } + + println!( + "{parsed} of {SCRIPTS} generated scripts parsed, {ran} compared, \ + {valued} produced a value, {skipped} hit a limit, \ + {walker_panics} panicked the walker, \ + {too_deep} were too complex to parse, \ + {lost_a_local} lost a local to rhai's optimizer", + ); + + // The walker panicking is upstream's problem, but it is also a hole in this + // test's coverage, so it is worth knowing if it ever becomes common. + assert!( + walker_panics * 50 < SCRIPTS, + "{walker_panics} of {SCRIPTS} scripts panicked the walker, which is too \ + many to keep skipping — see \ + `rhai_underflows_a_scope_index_under_its_own_optimizer`", + ); + + // A generator that mostly emits garbage would pass this test by comparing + // two identical parse failures a few thousand times. These are the numbers + // that say it is doing work, and they are what to look at first if this + // file ever stops finding anything. + // + // Invalid *syntax* is the thing to catch, and a raw parse rate does not + // catch it: a script rejected for exceeding rhai's complexity limit is + // well-formed, and how many do is a property of how much the generator + // packs into one script rather than of whether it can write the language. + // So the two are counted apart, and this is the one that means something. + assert!( + unparsed.is_empty(), + "{} of {SCRIPTS} scripts are not valid rhai, so the generator is \ + emitting syntax rather than testing it:{}", + unparsed.len(), + unparsed.iter().take(5).cloned().collect::(), + ); + assert!( + parsed * 10 >= SCRIPTS * 7, + "only {parsed} of {SCRIPTS} scripts parsed, and {too_deep} were rejected \ + as too complex — the generator is packing more into a script than rhai \ + will take, so most of what it writes is never run", + ); + // Against the whole corpus rather than against what ran, because the two + // move for opposite reasons. A script that errors is still compared — the + // variant, the position and the scope all have to match — so a *rising* + // error rate is usually the generator reaching further, and measuring + // against `ran` would read that as a regression and push it back toward + // scripts too simple to catch anything. What this is here to catch is the + // generator producing nothing that works at all. + assert!( + valued * 5 >= SCRIPTS, + "only {valued} of {SCRIPTS} scripts reached a value; the rest failed at \ + run time, which tests error parity and little else", + ); + assert!( + skipped * 20 < SCRIPTS, + "{skipped} of {SCRIPTS} scripts hit an engine limit, so the budgets are \ + deciding the outcome rather than the code", + ); + + assert!(failures.is_empty(), "generated scripts diverged:{}", failures.join(""),); +} diff --git a/tests/grain/limits.rs b/tests/grain/limits.rs new file mode 100644 index 000000000..c29c84911 --- /dev/null +++ b/tests/grain/limits.rs @@ -0,0 +1,162 @@ +//! Compiled code must still be stoppable. +//! +//! Rhai enforces `max_operations` and the `on_progress` interrupt from +//! `Engine::track_operation`, which the tree walker calls per AST node. A VM +//! that never called it would turn `loop {}` from a script the engine +//! terminates into one that hangs the host — a safety regression, not a +//! performance one, which is why `track_operation` is in the patch. +//! +//! These live outside the differential corpus on purpose. The walker ticks per +//! node and the VM ticks per loop back-edge, so the operation *counts* differ +//! and always will. What must hold is that the limit fires and the interrupt is +//! honoured, so that is what is asserted — not parity of counts or positions. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use rhai::grain::bytecode::Op; +use rhai::grain::{Compiler, Program, Vm}; +use rhai::{Dynamic, Engine, EvalAltResult, Scope}; + +/// A bare infinite loop, which the compiler lowers with nothing left over — +/// asserted below, so this cannot silently become a test of the fallback. +const SPIN: &str = "loop { }"; + +fn run_vm(engine: &Engine, source: &str) -> Result> { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert_eq!(program.residual_count(), 0, "{source:?} must be fully lowered, or this tests rhai rather than the VM",); + + // Without a tick on the back-edge nothing in a compiled loop ever reaches + // `track_operation`, and the tests below would hang rather than fail. + assert!(program.main().ops(program.code()).any(|(_, op)| op == Op::Tick), "{source:?} lowered to a loop with no operation tick",); + + Vm::new(engine).eval_with_scope(&mut Scope::new(), &program) +} + +#[test] +fn compiled_loop_hits_the_operation_limit() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let err = run_vm(&engine, SPIN).expect_err("an unbounded loop must be stopped"); + + assert!(matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), "expected ErrorTooManyOperations, got {err:?}",); +} + +#[test] +fn compiled_loop_honours_the_progress_interrupt() { + let ticks = Arc::new(AtomicU64::new(0)); + let seen = ticks.clone(); + + let mut engine = Engine::new(); + engine.on_progress(move |count| { + seen.store(count, Ordering::SeqCst); + // Stand-in for a host's abort flag. + (count >= 500).then(|| Dynamic::from("terminated")) + }); + + let err = run_vm(&engine, SPIN).expect_err("the interrupt must stop the loop"); + + assert!(matches!(*err, EvalAltResult::ErrorTerminated(..)), "expected ErrorTerminated, got {err:?}",); + assert!(ticks.load(Ordering::SeqCst) >= 500, "on_progress should have been called on every back-edge",); +} + +/// A chunk that loops with no tick in it must still be stopped. +/// +/// Every loop this compiler emits carries an `Op::Tick` on its back-edge, so +/// nothing it produces can spin. An artifact is not required to have come from +/// it. Turning this program's tick into a no-op leaves a chunk that still +/// verifies — the jump is in range, the stack balances, every path reaches a +/// `Return` — and runs forever, which makes the engine's budget the only thing +/// between a host and a hostile file. +/// +/// So the budget cannot depend on the compiler having been generous: the VM +/// charges an operation for every *backward* transfer, and a cycle always has +/// one. Found by `mutated_artifacts_load_or_fail_but_never_misbehave`, which +/// hung on a mutation rather than failing. +#[test] +fn a_loop_with_its_tick_removed_still_hits_the_limit() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let ast = engine.compile(SPIN).expect("must compile"); + let program = Compiler::new().compile(&ast); + + // Where the tick sits inside the code, and what the code looks like, so the + // same bytes can be found again inside the finished artifact. + let code = program.code().to_vec(); + let (tick_at, _) = program.main().ops(program.code()).find(|(_, op)| *op == Op::Tick).expect("the compiler ticks a loop"); + + let mut bytes = program.write().expect("a lowered program must write"); + let start = bytes.windows(code.len()).position(|window| window == code).expect("the artifact embeds the code verbatim"); + + // `Checkpoint` is the other one-byte instruction that does nothing to the + // stack, so this swap leaves every offset, jump target and position entry + // exactly where it was. Only the metering goes. + bytes[start + tick_at] = rhai::grain::bytecode::code::tag::CHECKPOINT; + + let tickless = Program::read(&bytes).expect("still a valid artifact"); + assert!(!tickless.main().ops(tickless.code()).any(|(_, op)| op == Op::Tick), "the tick should be gone, or this tests nothing",); + + let err = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &tickless).expect_err("a tickless loop must still be stopped"); + assert!(matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), "expected ErrorTooManyOperations, got {err:?}",); +} + +/// The walker and the VM must agree that the script *fails*, even though they +/// disagree about after how many operations. +#[test] +fn the_walker_agrees_the_loop_is_stopped() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let ast = engine.compile(SPIN).expect("must compile"); + let err = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect_err("rhai must stop it too"); + + assert!(matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), "expected ErrorTooManyOperations, got {err:?}",); +} + +/// `max_string_size` is a host's defence, and interpolation is the easiest way +/// to walk past it — rhai checks the running total after *every* segment +/// rather than once at the end, so a script cannot build a huge string and +/// hand it over. +/// +/// The position is checked too, because it is the one thing a single +/// instruction might not be able to reproduce: rhai blames the segment that +/// tipped the total over, and the VM has one position-table entry per +/// instruction. +#[test] +fn interpolation_respects_the_string_limit() { + let mut engine = Engine::new(); + engine.set_max_string_size(16); + + let source = r#"let a = "0123456789"; `${a}${a}${a}`"#; + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "must be lowered, not walked"); + + let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect_err("the walker must refuse it"); + let vm = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect_err("and so must the VM"); + + assert!(matches!(*vm, EvalAltResult::ErrorDataTooLarge(..)), "got {vm:?}",); + assert_eq!(format!("{vm:?}"), format!("{walker:?}"), "including the position of the segment that went over",); +} + +/// A loop that does terminate must not be killed by the tick itself, and must +/// still produce the value rhai produces. +#[test] +fn ticking_does_not_disturb_a_bounded_loop() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let source = "let i = 0; loop { i += 1; if i > 100 { break i; } }"; + let ast = engine.compile(source).expect("must compile"); + + let program = Compiler::new().compile(&ast); + let vm = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect("bounded loop must finish"); + + let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect("bounded loop must finish under rhai too"); + + assert_eq!(format!("{vm:?}"), format!("{walker:?}")); +} diff --git a/tests/grain/projection.rs b/tests/grain/projection.rs new file mode 100644 index 000000000..9ad8166ed --- /dev/null +++ b/tests/grain/projection.rs @@ -0,0 +1,225 @@ +//! What would a lowering of this script actually weigh? +//! +//! This tests the premise before an instruction set is built to serve it. +//! `follow.rhai` uses most of the language — nested index writes, property +//! chains, method calls, script functions, `loop`/`while`/`if-else` — so +//! lowering it for real is a lot of work. Doing that first and *then* +//! discovering the artifact is only marginally smaller than the tree would be +//! the expensive way to learn it. +//! +//! So this counts the real AST, node by node, and prices each node against the +//! planned stack encoding. The per-node costs are stated in `encoded_size` +//! where they can be argued with. +//! +//! **This is a projection, not a measurement.** It is a lower bound in one +//! direction and optimistic in another: it assumes every node lowers to the +//! ops listed, and it ignores the operand stack traffic a real lowering emits +//! for temporaries. `tests/grain/format.rs` measures the artifact that +//! actually results, and that number is the one to trust. + +use std::collections::BTreeSet; + +use rhai::{ASTNode, Engine, Expr, Stmt, AST}; + +const SOURCE: &str = include_str!("fixtures/follow.rhai"); + +/// Marks a node kind the model does not know how to price. Its presence is a +/// test failure: a silently-unpriced variant would understate the projection. +const UNPRICED: &str = "UNPRICED"; + +/// A node's kind, and what the planned encoding would spend on it. +/// +/// The encoding is a stack machine: one byte of opcode, then operands as +/// varints that are one byte at this program's scale (fewer than 128 locals, +/// constants or names). Costs below are in bytes. +fn encoded_size(node: &ASTNode) -> (&'static str, usize) { + match node { + ASTNode::Expr(expr) => match expr { + // Small integers get a dedicated opcode with an inline operand; + // anything else is an index into the constant pool. Either way two + // bytes, which is why they are not split here. + Expr::IntegerConstant(..) => ("int", 2), + Expr::FloatConstant(..) => ("float", 2), + Expr::StringConstant(..) => ("string", 2), + Expr::CharConstant(..) => ("char", 2), + Expr::DynamicConstant(..) => ("dynamic const", 2), + // No operand: the opcode is the value. + Expr::BoolConstant(..) => ("bool", 1), + Expr::Unit(..) => ("unit", 1), + Expr::ThisPtr(..) => ("this", 1), + + // LoadLocal + slot. The parser has already resolved most of these + // to a slot index, so this is not an optimistic assumption. + Expr::Variable(..) => ("variable", 2), + + // Operators become a typed opcode with no operands when both sides + // are primitives, and a generic call otherwise. Scored as a call, + // the pessimistic reading, because the deopt guard may disable the + // fast path for a given engine. + Expr::FnCall(x, ..) if x.op_token.is_some() => ("operator", 1), + Expr::FnCall(..) => ("call", 3), + Expr::MethodCall(..) => ("method call", 3), + + // Get by interned name; set is a separate op emitted by the parent. + Expr::Property(..) => ("property", 2), + Expr::Dot(..) => ("dot", 1), + Expr::Index(..) => ("index", 1), + + // Length-prefixed build from the operand stack. + Expr::Array(..) => ("array literal", 2), + Expr::Map(..) => ("map literal", 2), + Expr::InterpolatedString(..) => ("interpolation", 2), + + // Short-circuit: a conditional jump per operand. + Expr::And(..) => ("and", 3), + Expr::Or(..) => ("or", 3), + Expr::Coalesce(..) => ("coalesce", 3), + + // Block-as-expression needs no instruction of its own; the + // statements inside are counted separately. + Expr::Stmt(..) => ("block expr", 0), + + // Cannot be lowered at all — the handler is looked up by string + // against a live Engine. Priced as the residual it would stay. + Expr::Custom(..) => ("custom syntax", 3), + + // `Expr` is #[non_exhaustive]. A variant added by a rhai upgrade + // must surface as a test failure, not get quietly priced at zero. + _ => (UNPRICED, 0), + }, + + ASTNode::Stmt(stmt) => match stmt { + Stmt::Noop(..) => ("noop", 0), + // The block's own statements are counted as they are walked. + Stmt::Block(..) => ("block", 0), + + // StoreLocal + slot. + Stmt::Var(..) => ("var decl", 2), + // Store, plus the operator when compound. + Stmt::Assignment(..) => ("assignment", 3), + + // Condition jump plus the jump over the else arm. + Stmt::If(..) => ("if", 6), + // Condition jump plus the backward jump. + Stmt::While(..) => ("while", 6), + Stmt::Do(..) => ("do", 6), + // Iterator setup, plus a step-and-branch per turn. + Stmt::For(..) => ("for", 8), + // Jump table plus a linear range list. + Stmt::Switch(..) => ("switch", 12), + + // Jumps to a fixed target. + Stmt::BreakLoop(..) => ("break/continue", 3), + Stmt::Return(..) => ("return/throw", 3), + + // Handler region, registered rather than executed. + Stmt::TryCatch(..) => ("try/catch", 8), + + Stmt::FnCall(..) => ("call stmt", 3), + // Discard the value the expression left behind. + Stmt::Expr(..) => ("expr stmt", 1), + + #[cfg(not(feature = "no_module"))] + Stmt::Import(..) => ("import", 3), + #[cfg(not(feature = "no_module"))] + Stmt::Export(..) => ("export", 3), + #[cfg(not(feature = "no_closure"))] + Stmt::Share(..) => ("share", 2), + + _ => (UNPRICED, 0), + }, + + _ => (UNPRICED, 0), + } +} + +/// Names that would live in the artifact's string table, deduplicated. +/// +/// Only the ones reachable without unpacking every boxed payload: variables, +/// called functions and properties. That undercounts, so the projected string +/// table is a floor. +fn interned_names(ast: &AST) -> BTreeSet { + let mut names = BTreeSet::new(); + + ast.walk(&mut |path: &[ASTNode]| { + if let Some(ASTNode::Expr(expr)) = path.last() { + match expr { + Expr::Variable(x, ..) => { + names.insert(x.1.to_string()); + } + Expr::FnCall(x, ..) | Expr::MethodCall(x, ..) => { + names.insert(x.name.to_string()); + } + Expr::StringConstant(s, ..) => { + names.insert(s.to_string()); + } + _ => {} + } + } + true + }); + + names +} + +#[test] +fn projected_artifact_size() { + let engine = Engine::new(); + let ast = engine.compile(SOURCE).expect("fixtures/follow.rhai must compile"); + + let mut counts: std::collections::BTreeMap<&'static str, (usize, usize)> = std::collections::BTreeMap::new(); + let mut code_bytes = 0usize; + let mut nodes = 0usize; + + ast.walk(&mut |path: &[ASTNode]| { + let Some(node) = path.last() else { + return true; + }; + let (kind, size) = encoded_size(node); + let entry = counts.entry(kind).or_default(); + entry.0 += 1; + entry.1 += size; + code_bytes += size; + nodes += 1; + true + }); + + let names = interned_names(&ast); + // One length byte plus the UTF-8 bytes, per distinct name. + let string_table: usize = names.iter().map(|n| 1 + n.len()).sum(); + + // A tag byte plus eight bytes of payload, for every node that referenced + // the pool. Deduplication would shrink this; not modelling it keeps the + // projection on the pessimistic side. + let pool_refs: usize = ["int", "float", "string", "char", "dynamic const"].iter().filter_map(|k| counts.get(k)).map(|(n, _)| *n).sum(); + let const_pool = pool_refs * 9; + + // Header, ABI fingerprint, and section offsets. + const HEADER: usize = 64; + + let projected = HEADER + string_table + const_pool + code_bytes; + let source_bytes = SOURCE.len(); + + println!("\nfixtures/follow.rhai — {source_bytes} source bytes, {nodes} AST nodes\n"); + println!("{:<18} {:>7} {:>9}", "node kind", "count", "bytes"); + for (kind, (n, bytes)) in &counts { + println!("{kind:<18} {n:>7} {bytes:>9}"); + } + + println!("\n{:<18} {:>9}", "code", code_bytes); + println!("{:<18} {:>9} ({} distinct names)", "string table", string_table, names.len()); + println!("{:<18} {:>9} ({pool_refs} pool refs)", "constants", const_pool); + println!("{:<18} {:>9}", "header", HEADER); + println!("{:<18} {:>9}", "projected total", projected); + + println!("\nprojected artifact / source {:.2}x", projected as f64 / source_bytes as f64); + + assert!(nodes > 0, "the walk visited nothing"); + assert!(code_bytes > 0, "every node priced at zero means the model is broken",); + assert!( + !counts.contains_key(UNPRICED), + "{} nodes have no cost in the model, so the projection understates: \ + a rhai upgrade added an AST variant", + counts[UNPRICED].0, + ); +} diff --git a/tests/grain/scope.rs b/tests/grain/scope.rs new file mode 100644 index 000000000..a561f1f54 --- /dev/null +++ b/tests/grain/scope.rs @@ -0,0 +1,758 @@ +//! Variables the caller already had, which no slot can name. +//! +//! Slots are indices into the caller's `Scope` counted from a base taken when +//! the program starts, so everything the host put there beforehand sits below +//! every slot. Those reads and writes go by name instead, and this is where +//! that is held to rhai's behaviour — including the three places rhai looks +//! and the order it looks in. +//! +//! It matters more than the node count suggests: a script that reads anything +//! its host supplied used to be a fragment, and a program with a fragment in +//! it cannot be written to an artifact at all. + +// `on_var` carries rhai's "volatile, may change" marker rather than a real +// deprecation — the same one `eval_expression_tree_raw` carries. Registering a +// resolver is the only way to test that the VM consults one. +#![allow(deprecated)] + +// Only the engine is wanted here; the corpus scripts belong to the harnesses +// that run all of them. +use super::corpus; + +use rhai::grain::{Compiler, Vm}; +use rhai::{Dynamic, Engine, Module, Scope, INT}; + +/// What a run produced, in a form two runs can be compared on. +#[derive(Debug, PartialEq, Eq)] +struct Outcome { + result: Result, + scope: Vec<(String, String)>, +} + +fn capture(scope: &Scope, result: Result>) -> Outcome { + Outcome { + result: result.map(|value| format!("{value:?}")).map_err(|err| format!("{err:?}")), + scope: scope.iter_raw().map(|(name, _, value)| (name.to_string(), format!("{value:?}"))).collect(), + } +} + +/// Run `source` under rhai and under the VM, from the same starting scope, and +/// require they agree on the value, the error and what the scope holds after. +/// +/// `writable` is the point of the exercise rather than a detail: a program +/// that still fragments cannot cross a wire, so a case that passes while +/// fragmenting has proved nothing about the feature. +#[track_caller] +fn agree(source: &str, build: impl Fn(&mut Scope), writable: bool) { + agree_with(&corpus::engine(), source, build, writable); +} + +/// The same, against an engine the caller has set up — a variable resolver, a +/// published module. Both change what a name resolves to, which is most of what +/// this file is about. +#[track_caller] +fn agree_with(engine: &Engine, source: &str, build: impl Fn(&mut Scope), writable: bool) { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert_eq!(program.residual_count() == 0, writable, "{source:?} fragments: {:?}", program.first_unsupported(),); + + let mut walked = Scope::new(); + build(&mut walked); + let expected = capture(&walked.clone(), engine.eval_ast_with_scope::(&mut walked, &ast)); + let expected = Outcome { + scope: capture(&walked, Ok(Dynamic::UNIT)).scope, + ..expected + }; + + let mut run = Scope::new(); + build(&mut run); + let actual = { + let result = Vm::new(engine).eval_with_scope(&mut run, &program); + capture(&run, result) + }; + + assert_eq!(actual, expected, "{source:?}"); +} + +/// A script integer. Spelled through `INT` because `only_i32` narrows it, and +/// a caller variable holding the wider type is a host value no operator takes. +fn lit(value: INT) -> Dynamic { + Dynamic::from(value) +} + +#[test] +fn a_caller_variable_can_be_read() { + agree( + "brightness * 2", + |s| { + s.push("brightness", 21 as INT); + }, + true, + ); + agree( + "mode + \"!\"", + |s| { + s.push("mode", "chase".to_string()); + }, + true, + ); +} + +#[test] +fn a_caller_variable_can_be_written() { + agree( + "brightness = 7; brightness", + |s| { + s.push("brightness", 1 as INT); + }, + true, + ); + agree( + "brightness += 5; brightness", + |s| { + s.push("brightness", 1 as INT); + }, + true, + ); + // The op-assignment expansion path: no `-=` for strings, so rhai falls + // back to `x = x - y` and fails there rather than reporting no `-=`. + agree( + "mode += \"!\"; mode", + |s| { + s.push("mode", "go".to_string()); + }, + true, + ); +} + +/// Rhai's method-call rewrite reaches a caller's variable too. +/// +/// `f(x, ..)` means `x.f(..)`, so a `&mut` first parameter mutates the +/// variable. A local is addressed by slot and one the caller supplied is not, +/// but the rule is the same for both — which is the point, since whether a +/// script's variable is a local is not something the script says. +#[test] +fn a_caller_variable_in_first_argument_position_is_taken_by_reference() { + agree( + "push(log, 2); log", + |s| { + s.push("log", vec![lit(1)]); + }, + true, + ); + agree( + "bump(w); w.level", + |s| { + s.push("w", corpus::Widget::default()); + }, + true, + ); + // A constant is not a place rhai will hand out, so the mutation is + // discarded — and the caller's entry has to come back untouched. + agree( + "push(log, 2); log", + |s| { + s.push_constant("log", vec![lit(1)]); + }, + true, + ); + // Read after the other arguments, so it is the second name reported + // missing rather than the first. + agree("nosuch(gone, missing)", |_| {}, true); + agree( + "nosuch(gone, brightness)", + |s| { + s.push("brightness", 1 as INT); + }, + true, + ); +} + +/// A chain rooted at a caller's variable, which is the last root shape that +/// did not lower. +/// +/// Whether such a root can be written through is not known until it is looked +/// up, and rhai decides at the same moment and the same way: `search_namespace` +/// hands back a `Target`, and a scope entry becomes a reference where anything +/// else becomes a read-only value (`eval/expr.rs:120-155`). One case per arm of +/// that, because each fails differently. +#[test] +fn a_chain_can_be_rooted_at_a_caller_variable() { + let array = || vec![lit(1)]; + + // A writable entry: read, mutate through a method, and assign through. + agree( + "host[0]", + |s| { + s.push("host", array()); + }, + true, + ); + agree( + "host.push(2); host", + |s| { + s.push("host", array()); + }, + true, + ); + agree( + "host[0] = 9; host", + |s| { + s.push("host", array()); + }, + true, + ); + agree( + "host.level", + |s| { + s.push("host", corpus::Widget::default()); + }, + true, + ); + agree( + "host.level = 3; host.level", + |s| { + s.push("host", corpus::Widget::default()); + }, + true, + ); + + // A constant is not a place. Rhai refuses the assignment outright and + // refuses a mutating method too, because it never hands out a reference to + // one and a non-pure native will not take a read-only first argument + // (`func/call.rs:405`). + agree( + "host[0] = 9; host", + |s| { + s.push_constant("host", array()); + }, + true, + ); + agree( + "host.push(2); host", + |s| { + s.push_constant("host", array()); + }, + true, + ); + agree( + "host[0]", + |s| { + s.push_constant("host", array()); + }, + true, + ); + + // A shared entry walks through its cell's guard, so the mutation lands + // where every holder of the cell can see it. The closure is made in a + // block so the compared scope does not end up holding a pointer, which the + // two sides render differently on purpose. + agree( + "{ let keep = || host.len(); } host.push(2); host", + |s| { + s.push("host", array()); + }, + true, + ); + + // No entry at all, reported against the variable rather than the chain. + agree("nowhere[0]", |_| {}, true); + agree("nowhere.push(1)", |_| {}, true); +} + +/// The other two things a name can resolve to, neither of which is a place. +#[test] +fn a_chain_rooted_at_a_resolved_name_cannot_be_written_through() { + let mut engine = corpus::engine(); + engine.on_var(|name, _, _| { + Ok(match name { + "injected" => Some(Dynamic::from(vec![lit(7)])), + _ => None, + }) + }); + + let mut module = rhai::Module::new(); + module.set_var("published", vec![lit(5)]); + engine.register_global_module(module.into()); + + for source in [ + // A resolver's answer is read-only, so both the write and the mutating + // method are refused. + "injected[0]", + "injected[0] = 9; injected", + "injected.push(2); injected", + // A module's constant is not, so both go into a copy and are discarded. + // The two constants are not the same constant, which is the thing worth + // pinning here (`eval/expr.rs:151` against `:122`). + "published[0]", + "published[0] = 9", + "published.push(2)", + ] { + agree_with(&engine, source, |_| {}, true); + } +} + +/// A closure can capture a variable the caller supplied, and capturing it means +/// binding the cell rather than a copy of what is in it. +/// +/// Both halves of that were wrong. Sharing found the entry by walking +/// `Scope::iter_raw`, which runs from the top down, and used the position as if +/// it counted from the bottom — so with more than one caller entry it shared +/// the wrong variable outright. And the read that binds the capture went +/// through the flattening one, so even the right variable was captured by +/// value: a write afterwards was invisible to the closure. +#[test] +fn a_closure_can_capture_a_caller_variable() { + let seed = |scope: &mut Scope| { + // Two of them, and the interesting one is not last: the index bug is + // invisible with a single entry. + scope.push("first", vec![lit(7)]); + scope.push("second", vec![lit(1)]); + }; + + // The write happens after the closure is made, so a captured copy answers + // with the old length. + agree("let n = 0; { let f = || first.len(); first.push(9); n = f.call(); } n", seed, true); + agree("let n = 0; { let f = || second.len(); second.push(9); n = f.call(); } n", seed, true); + // And the capture is what shares it, which `is_shared` can see. + agree("{ let f = || first.len(); } is_shared(first)", seed, true); + agree("{ let f = || first.len(); } is_shared(second)", seed, true); +} + +/// A local of the same name hides the caller's and must not write through to +/// it — the caller's entry has to come back untouched. +#[test] +fn a_local_shadows_the_caller_without_disturbing_it() { + agree( + "let brightness = 1; brightness += 1; brightness", + |s| { + s.push("brightness", 100 as INT); + }, + true, + ); + // And the other order: read before the local exists, so the same name is + // two different variables in one script. + agree( + "let first = brightness; let brightness = 1; [first, brightness]", + |s| { + s.push("brightness", 100 as INT); + }, + true, + ); +} + +#[test] +fn a_caller_constant_cannot_be_assigned_to() { + let constant = |s: &mut Scope| { + s.push_constant("mode", lit(1)); + }; + + agree("mode = 2; mode", constant, true); + agree("mode += 1; mode", constant, true); + // Reading one is fine, which is the half that must keep working. + agree("mode + 1", constant, true); +} + +#[test] +fn a_name_that_is_nowhere_is_reported_the_same_way() { + agree("nope + 1", |_| {}, true); + agree("nope = 1", |_| {}, true); + agree("nope += 1", |_| {}, true); +} + +/// A bare script-function name is a function pointer with the calling +/// environment attached (`eval/expr.rs:71-99`), not a variable read, so it +/// must stay a fragment rather than becoming a name lookup that fails. +/// +/// Checked on the compiled program rather than by running it, because running +/// it currently disagrees with the walker for an unrelated reason: `execute` +/// forces `always_search_scope` whenever a program has any fragment, and that +/// flag makes rhai skip the function-pointer branch entirely +/// (`eval/expr.rs:62`). Reported separately — it predates named variables, and +/// the fix is about residuals rather than about this. +#[test] +fn a_script_function_name_is_not_a_variable() { + let engine = corpus::engine(); + let ast = engine.compile("fn helper() { 1 } let f = helper; f.call()").expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert!(program.residual_count() > 0, "a function name must not become a name lookup",); + assert!( + !rhai::grain::bytecode::disassemble(program.code()).any(|(.., op)| matches!(op, rhai::grain::bytecode::Op::LoadNamed(..))), + "nothing in {:?} may load `helper` by name", + program, + ); +} + +/// The last of the three places rhai looks: a constant a host published on a +/// module rather than in the scope. +#[test] +fn a_global_module_constant_resolves() { + let mut engine = corpus::engine(); + let mut module = Module::new(); + module.set_var("CHANNELS", 512 as INT); + engine.register_global_module(module.into()); + + let ast = engine.compile("CHANNELS / 2").expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0); + + let value = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect("a module constant must resolve"); + assert_eq!(value.as_int().unwrap(), 256); + + // And writing to one is refused, because it is a value and not a place. + let ast = engine.compile("CHANNELS = 1").expect("must compile"); + let program = Compiler::new().compile(&ast); + let err = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect_err("a module constant is not assignable"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), "got {err:?}",); +} + +/// A script `import` costs the whole body its lowering, and must. +/// +/// `import` declares into the imports stack, not the scope. A per-statement +/// fragment rewinds that stack on the way out (`eval/stmt.rs:55`), so the alias +/// would be dropped before the qualified call — itself a separate fragment — +/// could name it, and the VM answered `Module not found` where the walker +/// answered. That is a wrong result rather than a missing feature, which is the +/// one thing the fragment fallback is not allowed to produce. +/// +/// So the compiler refuses the lowering instead, and the walker takes the body +/// as one block. These cases are here to keep it refusing: an `import` that +/// started lowering again would put the divergence straight back. +#[test] +#[cfg(not(feature = "no_module"))] +fn an_import_keeps_the_walkers_answer() { + let mut engine = corpus::engine(); + + let mut resolver = rhai::module_resolvers::StaticModuleResolver::new(); + let module_ast = engine.compile("fn double(x) { x * 2 } export const LIMIT = 99;").expect("the module source must parse"); + let module = Module::eval_ast_as_new(Scope::new(), &module_ast, &engine).expect("the module must build"); + resolver.insert("kit", module); + engine.set_module_resolver(resolver); + + // An `import` in the body costs the body its lowering, so the program is a + // fragment rhai's walker has to evaluate and cannot become an artifact. + for source in [ + r#"import "kit" as k; k::double(21)"#, + r#"import "kit" as k; k::LIMIT"#, + r#"let r = 0; { import "kit" as k; r = k::double(4); } r"#, + r#"import "kit" as k; let t = 0; for i in 0..3 { t += k::double(i); } t"#, + ] { + agree_with(&engine, source, |_| {}, false); + } + + // In a function body the fallback is per-function: that body stays an AST + // in the library and the top level still lowers whole. + agree_with(&engine, r#"fn via_kit() { import "kit" as k; k::double(3) } via_kit()"#, |_| {}, true); +} + +/// `eval` costs the body its lowering, and must. +/// +/// It runs against the caller's scope and can declare into it, and the AST +/// does not say so. The slot model resolves indices against the scope shape it +/// can see, so a read after one looks in the wrong place — +/// `eval("let x = 40"); x + 2` found no `x` where the walker found 40. +/// +/// Written as a divergence rather than a missing feature, which is what makes +/// it worth a test: it used to become an ordinary fragment and answer wrongly. +#[test] +fn eval_keeps_the_walkers_answer() { + let engine = corpus::engine(); + + for source in [ + // The case that diverged: what `eval` declares outlives it. + r#"eval("let x = 40"); x + 2"#, + r#"let x = 1; eval("x = 99"); x"#, + r#"eval("1 + 1")"#, + r#"eval("let y = 7; y * 3")"#, + ] { + agree_with(&engine, source, |_| {}, false); + } +} + +/// The same for custom syntax, which reaches the caller's scope through an +/// `EvalContext` and is likewise invisible to the slot model. +#[test] +#[cfg(not(feature = "no_custom_syntax"))] +fn custom_syntax_keeps_the_walkers_answer() { + let mut engine = corpus::engine(); + engine + .register_custom_syntax(["declare", "$ident$", "=", "$int$"], true, |context, inputs| { + let name = inputs[0].get_string_value().unwrap().to_string(); + let value = inputs[1].get_literal_value::().unwrap(); + context.scope_mut().push(name, value); + Ok(Dynamic::UNIT) + }) + .expect("the custom syntax must register"); + + for source in [r#"declare foo = 41; foo + 1"#, r#"declare bar = 5; 10"#] { + agree_with(&engine, source, |_| {}, false); + } +} + +/// The first of the three, and the one a VM would most plausibly skip: a +/// resolver the host registered through `Engine::on_var` sees the name before +/// the scope does. +#[test] +fn a_variable_resolver_is_consulted_first() { + let mut engine = corpus::engine(); + engine.on_var(|name, _, _| { + Ok(match name { + "injected" => Some(lit(99)), + // Declining must fall through to the scope rather than fail. + _ => None, + }) + }); + + let compile = |engine: &Engine, source: &str| { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "{source:?} must not fragment"); + program + }; + + let mut scope = Scope::new(); + scope.push("ordinary", 5 as INT); + + let program = compile(&engine, "injected + ordinary"); + let value = Vm::new(&engine).eval_with_scope(&mut scope.clone(), &program).expect("both must resolve"); + assert_eq!(value.as_int().unwrap(), 104); + + // A resolver hands back a value rather than a place, so it is read-only. + let program = compile(&engine, "injected = 1"); + let err = Vm::new(&engine).eval_with_scope(&mut scope.clone(), &program).expect_err("a resolved variable is not assignable"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), "got {err:?}",); + + // And the walker agrees about all of it. + let ast = engine.compile("injected + ordinary").expect("must compile"); + let expected = engine.eval_ast_with_scope::(&mut scope.clone(), &ast).expect("the walker must agree"); + assert_eq!(expected.as_int().unwrap(), 104); +} + +/// A resolver that pushes onto the scope invalidates every parse-time index, +/// and rhai stops trusting them from that point. Nothing this compiler emits +/// depends on those, but a fragment's does — so the flag still has to be set. +#[test] +fn a_resolver_that_grows_the_scope_forces_a_search() { + let mut engine = corpus::engine(); + engine.on_var(|name, _, mut context| { + if name == "grow" { + context.scope_mut().push("added", 1 as INT); + return Ok(Some(lit(1))); + } + Ok(None) + }); + + let ast = engine.compile("let a = 10; grow + a").expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0); + + let mut scope = Scope::new(); + let value = Vm::new(&engine).eval_with_scope(&mut scope, &program).expect("must run"); + + let mut walked = Scope::new(); + let expected = engine.eval_ast_with_scope::(&mut walked, &ast).expect("the walker must run it too"); + + assert_eq!(format!("{value:?}"), format!("{expected:?}")); +} + +/// The receiver a resolver answered is not the scope entry of the same name. +/// +/// This is the case the by-reference rewrite is most easily got wrong on: the +/// resolver is consulted first and hands back a *value*, so `push(shadowed, 2)` +/// mutates a temporary and the caller's entry is left alone — even though there +/// is an entry of that name sitting right there to take a reference to. +/// +/// The VM tells them apart by the value already on the stack being read-only, +/// which is how `load_named` marks a resolver's answer. The alternative would +/// be running the resolver a second time, and a host can see that. +#[test] +fn a_resolved_receiver_is_not_the_scope_entry_it_shadows() { + let mut engine = corpus::engine(); + engine.on_var(|name, _, _| { + Ok(match name { + "shadowed" => Some(Dynamic::from(vec![lit(9)])), + _ => None, + }) + }); + + let source = "push(shadowed, 2); shadowed"; + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "the call must lower"); + + let start = |scope: &mut Scope| { + scope.push("shadowed", vec![lit(1)]); + }; + + let mut walked = Scope::new(); + start(&mut walked); + let walker = engine.eval_ast_with_scope::(&mut walked, &ast); + + let mut run = Scope::new(); + start(&mut run); + let ours = Vm::new(&engine).eval_with_scope(&mut run, &program); + + // Read-only all the way through, so rhai refuses the call outright rather + // than mutating a copy — which is a sharper thing to agree on. + assert!(matches!(walker.as_ref().unwrap_err().as_ref(), rhai::EvalAltResult::ErrorNonPureMethodCallOnConstant(..),), "got {walker:?}",); + assert_eq!(capture(&run, ours), capture(&walked, walker)); + assert_eq!(format!("{:?}", run.get_value::("shadowed").unwrap()), "[1]", "the entry the resolver shadowed must come back untouched",); +} + +/// The one place a compiled closure is not the walker's closure. +/// +/// Rhai's parser builds a pointer that embeds the closure's `ScriptFuncDef` — +/// the AST body — and tags it with the environment it was written in. That is +/// exactly what an artifact must not carry, so the compiler emits a +/// name-only pointer to the chunk it compiled from that same body. +/// +/// Everything the closure *does* is identical; what differs is that ours is +/// late-bound, which rhai renders. Pinned here rather than left to be +/// discovered, because it is visible to a script that prints one. +#[test] +fn a_closure_pointer_is_late_bound() { + let engine = corpus::engine(); + let source = "let n = 1; let f = |x| x + n; f"; + + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "the closure must lower"); + + let ours = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect("must run"); + let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect("must run under rhai too"); + + let (ours, walker) = (format!("{ours:?}"), format!("{walker:?}")); + assert!(ours.starts_with("Fn(\"anon$"), "ours is a plain named pointer: {ours}",); + assert!(walker.starts_with("Fn*+(\"anon$"), "rhai's carries a script body and an environment: {walker}",); + + // And the difference is only in the binding: calling either gives the + // same answer, which is what the corpus covers. + assert_ne!(ours, walker, "if these ever match, delete this test"); +} + +/// Calling a compiled function from outside, which is what a native wrapper +/// will do once compiled chunks are registered for callbacks. +#[test] +fn a_compiled_function_can_be_called_by_name() { + let engine = corpus::engine(); + let ast = engine.compile("fn add(a, b) { a + b } fn boom() { throw 7; } 0").expect("must compile"); + let program = Compiler::new().compile(&ast); + + let mut vm = Vm::new(&engine); + let value = vm.call_function(&program, "add", vec![lit(2), lit(3)], 0, rhai::Position::NONE).expect("must call"); + assert_eq!(value.as_int().unwrap(), 5); + + // Wrong arity is a miss, not a crash — the table is keyed on both. + let err = vm + .call_function(&program, "add", vec![lit(1)], 0, rhai::Position::NONE) + .expect_err("one argument is a different function"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorFunctionNotFound(..))); + + // And what the function raises comes back, wrapped as rhai wraps it. + let err = vm.call_function(&program, "boom", Vec::new(), 0, rhai::Position::NONE).expect_err("must propagate"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorInFunctionCall(..)), "got {err:?}",); + + // The operand stack is where it started, so a caller can keep using it. + let value = vm.call_function(&program, "add", vec![lit(10), lit(1)], 0, rhai::Position::NONE).expect("must call again"); + assert_eq!(value.as_int().unwrap(), 11); +} + +/// The same call through the API a host actually reaches for, which mirrors +/// [`Engine::call_fn`]. +/// +/// The differences from `call_function` are the ones rhai's own `call_fn` has: +/// arguments as a tuple rather than a `Vec`, a typed result, and the +/// program's body run first — which is what sets up anything the call needs. +#[test] +fn call_fn_mirrors_the_engines() { + let engine = corpus::engine(); + let ast = engine.compile("fn add(a, b) { a + b } let marker = 10; 0").expect("must compile"); + let program = Compiler::new().compile(&ast); + + let mut vm = Vm::new(&engine); + let mut scope = Scope::new(); + + let sum: INT = vm.call_fn(&mut scope, &program, "add", (2 as INT, 3 as INT)).expect("must call"); + assert_eq!(sum, 5); + // `eval_ast` ran the body, and `rewind_scope` took back what it declared. + // Both default to on, as they do in rhai. + assert_eq!(scope.len(), 0, "the default rewinds what the body declared"); + + let keep = rhai::CallFnOptions::new().rewind_scope(false); + let sum: INT = vm.call_fn_with_options(keep, &mut scope, &program, "add", (4 as INT, 5 as INT)).expect("must call"); + assert_eq!(sum, 9); + assert_eq!(scope.len(), 1, "`marker` outlives the call now"); + assert_eq!(scope.get_value::("marker").unwrap(), 10); + + // Skipping the body is what a caller does when the function needs nothing + // from it — the common case, and the one that costs nothing. + let mut fresh = Scope::new(); + let skip = rhai::CallFnOptions::new().eval_ast(false); + let sum: INT = vm.call_fn_with_options(skip, &mut fresh, &program, "add", (1 as INT, 1 as INT)).expect("must call"); + assert_eq!(sum, 2); + assert_eq!(fresh.len(), 0, "the body never ran, so it declared nothing"); + + // A result that is not the type asked for is an error, not a panic. + let err = vm.call_fn::(&mut scope, &program, "add", (1 as INT, 2 as INT)).expect_err("an INT is not a String"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorMismatchOutputType(..)), "got {err:?}",); + + // And a missing name still misses. + let err = vm.call_fn::(&mut scope, &program, "nope", (1 as INT,)).expect_err("no such function"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorFunctionNotFound(..)), "got {err:?}",); +} + +/// The flag a host uses to decide whether a program has to be owned. +/// +/// Registering compiled functions so a native can call one back requires a +/// `'static` wrapper, which means owning the program and giving up the +/// borrowed-from-the-artifact loading. Nobody should have to read a script to +/// find out whether that is needed — and the answer must be the same for a +/// compiled program and for the same program read back, or one of the two +/// paths quietly loses its callbacks. +#[test] +fn the_compiler_says_whether_a_program_makes_function_pointers() { + let engine = corpus::engine(); + + for (source, expected) in [ + ("let a = 1; a + 2", false), + ("fn f(x) { x } f(1)", false), + ("let s = 0; for i in 0..3 { s += i; } s", false), + // Every shape that produces one. + ("let n = \"ab\" + \"s\"; Fn(n)", true), + ("let n = 1; let f = |x| x + n; f.call(2)", true), + ("fn t(x) { x } let f = Fn(\"t\"); f.call(1)", true), + ("fn a(x, y) { x } let n = \"a\"; Fn(n).curry(1)", true), + ] { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.makes_fn_pointers(), expected, "for {source:?}",); + + // Read off the code, so an artifact answers the same. + if let Ok(bytes) = program.write() { + let reloaded = rhai::grain::Program::read(&bytes).expect("must load"); + assert_eq!(reloaded.makes_fn_pointers(), expected, "after a round trip, for {source:?}",); + } + } +} + +/// The whole reason for the opcode: these now cross a wire. +#[test] +fn a_program_reading_caller_state_can_be_written() { + let engine = corpus::engine(); + let source = "let out = brightness; if mode == \"chase\" { out += 10 } out"; + + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + let bytes = program.write().expect("must be writable"); + + let reloaded = rhai::grain::Program::read(&bytes).expect("must load"); + + let mut scope = Scope::new(); + scope.push("brightness", 5 as INT); + scope.push("mode", "chase".to_string()); + let value = Vm::new(&engine).eval_with_scope(&mut scope, &reloaded).expect("must run"); + + assert_eq!(value.as_int().unwrap(), 15); +} diff --git a/tests/mod.rs b/tests/mod.rs new file mode 100644 index 000000000..90168b007 --- /dev/null +++ b/tests/mod.rs @@ -0,0 +1,24 @@ +mod grain { + // Declared once here rather than in each harness: they are modules of this + // binary, not crate roots, so a `mod corpus;` of their own would look for + // `/corpus.rs`. + mod corpus; + + // `allocation` is deliberately absent: it owns a counting global allocator + // and is its own binary, declared in Cargo.toml. + mod call_fn; + mod callback; + mod differential; + mod format; + // Both are about execution staying inside a bound, which `unchecked` + // removes outright — and the artifact fuzzer needs `max_operations` to stop + // a corrupted chunk looping forever rather than failing. + #[cfg(not(feature = "unchecked"))] + mod fuzz; + #[cfg(not(feature = "unchecked"))] + mod limits; + // Prices rhai's own AST nodes, which are exported under `internals` only. + #[cfg(feature = "internals")] + mod projection; + mod scope; +} diff --git a/tests/native.rs b/tests/native.rs index e2579482c..3a07e1029 100644 --- a/tests/native.rs +++ b/tests/native.rs @@ -53,3 +53,28 @@ fn test_native_overload() { assert_eq!(engine.eval::(r#"let x = "hello"; let y = "world"; x + y"#).unwrap(), "hello***world"); assert_eq!(engine.eval::(r#"let x = "hello"; let y = (); x + y"#).unwrap(), "hello Foo!"); } + +/// A native asking rhai for a function it answers by name rather than by +/// dispatch. +/// +/// `type_of` and `is_shared` have no registered implementation anywhere, and a +/// call by name for one used to route past the code that implements them — +/// they are reserved names, and being reserved is what makes such a call +/// native-only. So a host could ask for `type_of` and be told there is no such +/// function, while the same question written in a script answered fine. +#[test] +fn test_native_call_fn_raw_reaches_syntactic_functions() { + let mut engine = Engine::new(); + + engine.register_raw_fn("ask_type", [TypeId::of::()], |context, args| { + let mut value = args[0].clone(); + context.call_fn_raw("type_of", false, false, &mut [&mut value]) + }); + + // A string, because its type name is the same on every build — `only_i32` + // and `f32_float` rename the numeric ones, and `no_index` removes arrays. + assert_eq!(engine.eval::(r#"let s = "a"; ask_type(s)"#).unwrap(), "string"); + + // And the answer is the script spelling's, whatever the numeric build is. + assert_eq!(engine.eval::("let x = 1; ask_type(x)").unwrap(), engine.eval::("let x = 1; type_of(x)").unwrap(),); +}