From 0daf56b3aab3fc4d1f8ee9565a850aa46a7b353e Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 13:40:18 -0700 Subject: [PATCH 01/33] initial code compiles --- .gitignore | 1 + Cargo.toml | 4 +- fuzz/Cargo.toml | 27 +- fuzz/fuzz_targets/generated.rs | 163 ++ fuzz/fuzz_targets/load.rs | 46 + fuzz/fuzz_targets/roundtrip.rs | 155 ++ src/eval/chaining.rs | 2 +- src/grain/bytecode/chain.rs | 195 +++ src/grain/bytecode/chunk.rs | 68 + src/grain/bytecode/code.rs | 929 ++++++++++ src/grain/bytecode/mod.rs | 18 + src/grain/bytecode/op.rs | 496 ++++++ src/grain/bytecode/positions.rs | 241 +++ src/grain/bytecode/strings.rs | 215 +++ src/grain/bytecode/switch.rs | 284 +++ src/grain/bytecode/verify.rs | 906 ++++++++++ src/grain/compile/cases.rs | 296 ++++ src/grain/compile/mod.rs | 1996 +++++++++++++++++++++ src/grain/compile/poolable.rs | 57 + src/grain/compile/slots.rs | 48 + src/grain/format/abi.rs | 235 +++ src/grain/format/mod.rs | 315 ++++ src/grain/format/read.rs | 453 +++++ src/grain/format/write.rs | 426 +++++ src/grain/mod.rs | 41 + src/grain/pos/mod.rs | 321 ++++ src/grain/pos/varint.rs | 136 ++ src/grain/program.rs | 590 +++++++ src/grain/vm/callback.rs | 130 ++ src/grain/vm/mod.rs | 2825 ++++++++++++++++++++++++++++++ src/lib.rs | 2 + src/module/mod.rs | 8 +- tests/grain/allocation.rs | 385 ++++ tests/grain/callback.rs | 216 +++ tests/grain/corpus/generate.rs | 562 ++++++ tests/grain/corpus/mod.rs | 663 +++++++ tests/grain/differential.rs | 363 ++++ tests/grain/fixtures/follow.rhai | 296 ++++ tests/grain/fixtures/golden.rgrn | Bin 0 -> 1759 bytes tests/grain/fixtures/golden.rhai | 92 + tests/grain/format.rs | 768 ++++++++ tests/grain/fuzz.rs | 502 ++++++ tests/grain/limits.rs | 219 +++ tests/grain/patch_integrity.rs | 32 + tests/grain/projection.rs | 242 +++ tests/grain/scope.rs | 638 +++++++ 46 files changed, 16599 insertions(+), 8 deletions(-) create mode 100644 fuzz/fuzz_targets/generated.rs create mode 100644 fuzz/fuzz_targets/load.rs create mode 100644 fuzz/fuzz_targets/roundtrip.rs create mode 100644 src/grain/bytecode/chain.rs create mode 100644 src/grain/bytecode/chunk.rs create mode 100644 src/grain/bytecode/code.rs create mode 100644 src/grain/bytecode/mod.rs create mode 100644 src/grain/bytecode/op.rs create mode 100644 src/grain/bytecode/positions.rs create mode 100644 src/grain/bytecode/strings.rs create mode 100644 src/grain/bytecode/switch.rs create mode 100644 src/grain/bytecode/verify.rs create mode 100644 src/grain/compile/cases.rs create mode 100644 src/grain/compile/mod.rs create mode 100644 src/grain/compile/poolable.rs create mode 100644 src/grain/compile/slots.rs create mode 100644 src/grain/format/abi.rs create mode 100644 src/grain/format/mod.rs create mode 100644 src/grain/format/read.rs create mode 100644 src/grain/format/write.rs create mode 100644 src/grain/mod.rs create mode 100644 src/grain/pos/mod.rs create mode 100644 src/grain/pos/varint.rs create mode 100644 src/grain/program.rs create mode 100644 src/grain/vm/callback.rs create mode 100644 src/grain/vm/mod.rs create mode 100644 tests/grain/allocation.rs create mode 100644 tests/grain/callback.rs create mode 100644 tests/grain/corpus/generate.rs create mode 100644 tests/grain/corpus/mod.rs create mode 100644 tests/grain/differential.rs create mode 100644 tests/grain/fixtures/follow.rhai create mode 100644 tests/grain/fixtures/golden.rgrn create mode 100644 tests/grain/fixtures/golden.rhai create mode 100644 tests/grain/format.rs create mode 100644 tests/grain/fuzz.rs create mode 100644 tests/grain/limits.rs create mode 100644 tests/grain/patch_integrity.rs create mode 100644 tests/grain/projection.rs create mode 100644 tests/grain/scope.rs diff --git a/.gitignore b/.gitignore index 77c1ef44e..d7b55da02 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ doc/rhai.json .idea .idea/* src/eval/chaining.rs +.zed/ diff --git a/Cargo.toml b/Cargo.toml index a0e1f789e..702331720 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = [".", "codegen", "codegen/tests/custom_root"] name = "rhai" version = "1.25.1" rust-version = "1.66.0" -edition = "2018" +edition = "2021" resolver = "2" authors = ["Jonathan Turner", "Lukáš Hozda", "Stephen Chung", "jhwgh1968"] description = "Embedded scripting for Rust" @@ -71,6 +71,8 @@ debugging = ["internals"] bin-features = ["decimal", "metadata", "serde", "debugging", "rustyline"] ## Enable fuzzing via the [`arbitrary`](https://crates.io/crates/arbitrary) crate. fuzz = ["arbitrary", "rust_decimal?/rust-fuzz", "serde"] +## Enable the experimental `grain` bytecode VM +grain = [] #! ### System Configuration Features diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index e7d47fb0f..1f8e311ac 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true anyhow = "1.0.78" arbitrary = { version = "1.3.2", features = ["derive"] } libfuzzer-sys = "0.4" -rhai = { path = "..", features = ["fuzz", "decimal", "metadata", "debugging"] } +rhai = { path = "..", features = ["fuzz", "decimal", "metadata", "debugging", "grain"] } serde = { version = "1.0.194", features = ["derive"] } # Prevent this from interfering with workspaces @@ -19,7 +19,9 @@ serde = { version = "1.0.194", features = ["derive"] } members = ["."] [profile.release] -debug = 1 +debug = true +debug-assertions = true +overflow-checks = true [[bin]] name = "scripting" @@ -38,3 +40,24 @@ name = "fuzz_serde" path = "fuzz_targets/fuzz_serde.rs" test = false doc = false + +[[bin]] +name = "load" +path = "fuzz_targets/load.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "roundtrip" +path = "fuzz_targets/roundtrip.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "generated" +path = "fuzz_targets/generated.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/generated.rs b/fuzz/fuzz_targets/generated.rs new file mode 100644 index 000000000..1f274a7a3 --- /dev/null +++ b/fuzz/fuzz_targets/generated.rs @@ -0,0 +1,163 @@ +//! Grammar-directed scripts through the pipeline, checked against the walker. +//! +//! `roundtrip` hands libfuzzer's bytes straight to the parser, which for a +//! language with real syntax means most of them are rejected before anything +//! interesting runs. This spends the same bytes on *grammar decisions* instead, +//! so every input is a valid script and coverage feedback is steering the shape +//! of the program rather than the spelling of it. +//! +//! `tests/fuzz.rs` runs the same generator over a few thousand seeded scripts +//! on every `cargo test`. This is the one that runs for hours. +//! +//! `cargo fuzz run generated` + +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rhai::grain::{Compiler, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +// The generator lives with the tests because that is the only thing that needs +// it, and putting it in the library would make it public API. This crate is +// outside the workspace and cannot depend on a test target, so it takes the +// source directly — inside a module, because the file opens with `//!`. +mod generate { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../tests/corpus/generate.rs" + )); +} + +use generate::Generator; + +/// Both sides budgeted identically, or a script that runs out of operations on +/// one and not the other reads as a divergence. +fn engine() -> Engine { + let mut engine = Engine::new(); + engine.set_max_operations(200_000); + engine.set_max_string_size(8192); + engine.set_max_array_size(2048); + engine.set_max_map_size(64); + // Pinned for the reason `tests/fuzz.rs` pins them: rhai's defaults are + // `debug_assertions`-gated (`api/limits.rs:10-36`), so an unpinned harness + // explores one script space here and a different one under `cargo test`. + // Same numbers in both, so a finding from one reproduces in the other. + engine.set_max_expr_depths(64, 64); + engine.set_max_call_levels(64); + engine +} + +/// Run the walker, surviving a panic of its own making. +/// +/// Rhai's optimizer can leave a local's parse-time scope index stale, and +/// evaluating it then underflows (`eval/expr.rs:131`). It is reachable from an +/// ordinary script with none of this involved — `tests/fuzz.rs` pins a +/// reproducer — and a fuzzing run trips it often enough to end the run in +/// minutes, which costs far more than it finds. There is nothing to compare +/// against a side that did not finish, so those inputs are dropped. +/// +/// The hook is swapped for the duration because libfuzzer installs one that +/// aborts, which would defeat `catch_unwind`. It is swapped back immediately, +/// so a panic anywhere else in this target still ends the run as it should. +fn walk(engine: &Engine, ast: &rhai::AST, scope: &mut Scope) -> Option { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + engine.eval_ast_with_scope::(scope, ast) + })); + std::panic::set_hook(hook); + result.ok() +} + +type RhaiOutcome = Result>; + +/// A run reduced to something two of them can be compared on, scope included: +/// a VM that produced the right value while leaving the scope a different shape +/// has still got it wrong, and that is how the slot model fails. +fn outcome(scope: &Scope, result: RhaiOutcome) -> Option { + let scope: Vec = scope + .iter_raw() + .map(|(name, _, value)| format!("{name}={value:?}")) + .collect(); + + match result { + Ok(value) => Some(format!("ok {value:?} [{}]", scope.join(","))), + // The limits neither side counts towards in lockstep. `tests/fuzz.rs` + // documents why and asserts they stay rare; here they are simply + // dropped, because there is no way to assert a rate over one input. + Err(err) + if matches!( + *err, + rhai::EvalAltResult::ErrorTooManyOperations(..) + | rhai::EvalAltResult::ErrorStackOverflow(..) + | rhai::EvalAltResult::ErrorTooManyVariables(..) + ) => + { + None + } + Err(err) => Some(format!("err {err:?} [{}]", scope.join(","))), + } +} + +/// Run a script both ways under one engine. `None` if there is nothing to +/// compare — it did not parse, the walker did not survive it, or a limit +/// decided it. +fn compare(engine: &Engine, source: &str) -> Option<(String, String)> { + let ast = engine.compile(source).ok()?; + + let mut walker_scope = Scope::new(); + let walked = walk(engine, &ast, &mut walker_scope)?; + let expected = outcome(&walker_scope, walked)?; + + let program = Compiler::new().compile(&ast); + let mut vm_scope = Scope::new(); + let ours = if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(engine).run_with_callbacks(&program, &mut vm_scope) + } else { + Vm::new(engine).run(&program, &mut vm_scope) + }; + Some((outcome(&vm_scope, ours)?, expected)) +} + +/// Whether a divergence is rhai's optimizer losing a local rather than a bug of +/// ours — see `rhai_drops_a_local_its_optimizer_still_refers_to` in +/// `tests/fuzz.rs`. +/// +/// The optimizer can delete a `let` whose variable is still read, leaving the +/// read pointing at whatever now sits at that scope index. Rhai answers with +/// another variable's value; we resolve by name and report it missing. There is +/// no agreeing with an AST that refers to a local it does not declare. +/// +/// Turning the optimizer off is what tells the two apart, and it runs only on a +/// divergence that already looks like this one — a fuzzer that quietly stopped +/// comparing would be worse than one that stops. +fn optimizer_lost_a_local(source: &str, ours: &str) -> bool { + if !ours.contains("ErrorVariableNotFound") { + return false; + } + let mut plain = engine(); + plain.set_optimization_level(rhai::OptimizationLevel::None); + compare(&plain, source).is_some_and(|(ours, expected)| ours == expected) +} + +fuzz_target!(|data: &[u8]| { + // Too few bytes to steer with, and the PRNG fallback would make every such + // input the same script. + if data.len() < 8 { + return; + } + + let source = Generator::from_bytes(data).script(); + let Some((ours, expected)) = compare(&engine(), &source) else { + return; + }; + if ours == expected || optimizer_lost_a_local(&source, &ours) { + return; + } + + assert_eq!( + ours, expected, + "the VM disagrees with the walker on:\n{source}" + ); +}); diff --git a/fuzz/fuzz_targets/load.rs b/fuzz/fuzz_targets/load.rs new file mode 100644 index 000000000..5ea94dc83 --- /dev/null +++ b/fuzz/fuzz_targets/load.rs @@ -0,0 +1,46 @@ +//! Arbitrary bytes into `Program::read`, then run whatever survives. +//! +//! This is the whole untrusted surface: a device is handed an artifact over a +//! link and executes it in place. The claim is total — any byte string either +//! fails to load, or loads into a chunk that runs without panicking, reading +//! outside itself, or running away. +//! +//! Running the survivors is the point rather than a bonus. A loader that +//! accepts a chunk it should not have has done nothing observable until +//! something executes it, so a target that only called `read` would miss the +//! failures that matter most. +//! +//! `cargo fuzz run load` + +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rhai::grain::{Program, Vm}; +use rhai::{Engine, Scope}; + +fuzz_target!(|data: &[u8]| { + let Ok(program) = Program::read(data) else { + return; + }; + + // `read` verifies before returning, and the VM's missing bounds checks + // rest on that. If it ever hands back something unverified, the contract + // is broken whether or not running it happens to work. + assert!( + program.verify().is_ok(), + "read returned a chunk that does not verify", + ); + + // Verification proves structure, not termination: a jump target that is + // in range is a valid infinite loop. That is not a gap a loader can close + // — no one can decide halting — which is why `Op::Tick` sits on every back + // edge and why a host running untrusted bytecode must set this. + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + engine.set_max_string_size(4096); + engine.set_max_array_size(1024); + engine.set_max_map_size(64); + engine.set_max_call_levels(16); + + let _ = Vm::new(&engine).run(&program, &mut Scope::new()); +}); diff --git a/fuzz/fuzz_targets/roundtrip.rs b/fuzz/fuzz_targets/roundtrip.rs new file mode 100644 index 000000000..2d665d8a2 --- /dev/null +++ b/fuzz/fuzz_targets/roundtrip.rs @@ -0,0 +1,155 @@ +//! Arbitrary *scripts* through the whole pipeline, checked against the walker. +//! +//! The other target proves a hostile artifact cannot misbehave. This one +//! proves a well-formed one still means what it said: compile, write, read +//! back, run, and get what rhai's own evaluator got — the differential corpus +//! generalised from cases someone thought of to inputs nobody did. +//! +//! Both halves are load-bearing and they fail differently. A divergence here +//! is a compiler or VM bug; a panic there is a safety bug. +//! +//! `cargo fuzz run roundtrip` + +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rhai::grain::{Compiler, Program, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +/// Both sides must be budgeted identically, or a script that runs out of +/// operations on one and not the other reads as a divergence. +fn engine() -> Engine { + let mut engine = Engine::new(); + engine.set_max_operations(50_000); + engine.set_max_string_size(4096); + engine.set_max_array_size(1024); + engine.set_max_map_size(64); + engine.set_max_call_levels(16); + // Parse-time depth is `debug_assertions`-gated in rhai + // (`api/limits.rs:10-36`), so without this the set of scripts this target + // accepts depends on how it was built. Call depth above is already pinned + // for the same reason. + engine.set_max_expr_depths(64, 64); + engine +} + +/// A value, with the one difference between the two sides that is intended. +/// +/// Rhai renders a closure pointer `Fn*+("anon$..")` — a script function with a +/// captured environment attached — and ours `Fn("anon$..")`, because ours is +/// name-only and resolved at call time. That is the whole point: a `Script` +/// pointer carries an AST, and an AST is what an artifact must not contain. +/// The difference is deliberate, pinned by `a_closure_pointer_is_late_bound` in +/// `tests/scope.rs`, and left in the rendering rather than papered over there. +/// +/// Here it has to be papered over, or the first script whose value is a bare +/// closure ends the run — which is within a few thousand executions. +fn rendered(value: &Dynamic) -> String { + format!("{value:?}") + .replace("Fn*+(", "Fn(") + .replace("Fn*(", "Fn(") +} + +/// A run reduced to something two of them can be compared on, scope included: +/// a VM that produced the right value while leaving the scope a different +/// shape has still got it wrong, and that is how the slot model fails. +fn outcome(run: impl FnOnce(&mut Scope) -> Result>) -> String { + let mut scope = Scope::new(); + let result = run(&mut scope); + let scope: Vec = scope + .iter_raw() + .map(|(name, _, value)| rendered(value)) + .collect(); + + match result { + Ok(value) => format!("ok {} [{}]", rendered(&value), scope.join(",")), + // The limits are the one thing allowed to differ: the walker ticks per + // AST node and the VM per loop back-edge, and a callback crossing costs + // the two of them different numbers of call levels. Neither count will + // ever match, so a script that runs into one is dropped. + Err(err) + if matches!( + *err, + rhai::EvalAltResult::ErrorTooManyOperations(..) + | rhai::EvalAltResult::ErrorStackOverflow(..) + | rhai::EvalAltResult::ErrorTooManyVariables(..) + ) => + { + "budget".to_string() + } + Err(err) => format!("err {err:?} [{}]", scope.join(",")), + } +} + +/// Whether a divergence is rhai's optimizer losing a local rather than a bug of +/// ours — see `rhai_drops_a_local_its_optimizer_still_refers_to` in +/// `tests/fuzz.rs`. +/// +/// The optimizer can delete a `let` whose variable is still read, leaving the +/// read pointing at whatever now sits at that scope index. Rhai answers with +/// another variable's value; we resolve by name and report it missing. There is +/// no agreeing with an AST that refers to a local it does not declare. +/// +/// Turning the optimizer off is what tells the two apart, and it runs only on a +/// divergence that already looks like this one — a fuzzer that quietly stopped +/// comparing would be worse than one that stops. +fn optimizer_lost_a_local(source: &str, direct: &str) -> bool { + if !direct.contains("ErrorVariableNotFound") { + return false; + } + let mut plain = engine(); + plain.set_optimization_level(rhai::OptimizationLevel::None); + let Ok(ast) = plain.compile(source) else { + return false; + }; + let program = Compiler::new().compile(&ast); + + let expected = outcome(|scope| plain.eval_ast_with_scope::(scope, &ast)); + let ours = outcome(|scope| Vm::new(&plain).run(&program, scope)); + ours == expected +} + +fuzz_target!(|source: String| { + let engine = engine(); + let Ok(ast) = engine.compile(&source) else { + return; + }; + + let program = Compiler::new().compile(&ast); + + let expected = outcome(|scope| engine.eval_ast_with_scope::(scope, &ast)); + // A program that can hand a pointer to a native has to be run the way such + // a program is meant to be run, or every one of them reads as a divergence. + let shared = program + .makes_fn_pointers() + .then(|| Compiler::new().compile(&ast).into_shared()); + let run = |scope: &mut Scope| match &shared { + Some(shared) => Vm::new(&engine).run_with_callbacks(shared, scope), + None => Vm::new(&engine).run(&program, scope), + }; + + let direct = outcome(run); + if expected == "budget" || direct == "budget" { + return; + } + if direct != expected && optimizer_lost_a_local(&source, &direct) { + return; + } + assert_eq!( + direct, expected, + "the VM disagrees with the walker on:\n{source}" + ); + + // A program that still fragments cannot be written, which is not a bug — + // it is the escape hatch doing its job. + let Ok(bytes) = program.write() else { + return; + }; + let reloaded = Program::read(&bytes).expect("what we wrote must read back"); + let loaded = outcome(|scope| Vm::new(&engine).run(&reloaded, scope)); + + assert_eq!( + loaded, expected, + "the artifact disagrees with the walker on:\n{source}" + ); +}); diff --git a/src/eval/chaining.rs b/src/eval/chaining.rs index 7c19eaf73..d90d64923 100644 --- a/src/eval/chaining.rs +++ b/src/eval/chaining.rs @@ -108,7 +108,7 @@ impl Engine { /// Panics if the target object is shared. /// /// Shared objects should be handled (dereferenced) before calling this method. - fn get_indexed_mut<'t>( + pub(crate) fn get_indexed_mut<'t>( &self, global: &mut GlobalRuntimeState, caches: &mut Caches, diff --git a/src/grain/bytecode/chain.rs b/src/grain/bytecode/chain.rs new file mode 100644 index 000000000..b4a237834 --- /dev/null +++ b/src/grain/bytecode/chain.rs @@ -0,0 +1,195 @@ +/// One step along `a.b[i].c(x)`. +/// +/// Steps live in the program's chain pool rather than in the instruction +/// stream, because a chain is walked by one instruction rather than several. +/// It has to be: the walk holds a `&mut` into the container at every level, and +/// a borrow cannot survive a trip round the dispatch loop. That is also what +/// makes it correct — rhai holds the same references, so a mutation partway +/// down a chain lands in the same place rather than in a copy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Step { + /// `[i]`, where the index was evaluated onto the operand stack before the + /// chain instruction ran, at `operand` from the first of them. + /// + /// Pre-evaluating mirrors rhai, which collects every index in a chain into + /// `idx_values` before walking it (`eval/chaining.rs:568`). It has to + /// happen first: evaluating an index halfway down would need the operand + /// stack while a borrow of the container is live. + /// + /// Rhai reports `a[10]` out of bounds against the `10` rather than against + /// the chain (`eval/chaining.rs:694`). + /// + /// `bracket` is the other position rhai keeps for a step, and the two are + /// not interchangeable: `pos` is where the index expression starts and + /// `bracket` is the `[` in front of it (`op_pos`, `eval/chaining.rs:695`). + /// An out-of-bounds index is blamed on the first and indexing something + /// that cannot be indexed on the second, so `a[0][5]` where `a[0]` is not + /// indexable names the *second* `[` — the step that failed — and a chain + /// carrying one position between it and its neighbours would name the + /// wrong one. + Index { + operand: u16, + pos: rhai::Position, + bracket: rhai::Position, + }, + + /// `.name`, which is a key lookup on a map and a getter call on anything + /// else — the distinction rhai makes at runtime, not at parse time + /// (`eval/chaining.rs:898`). + Property { + /// The bare name, for a map key and for error messages. + name: u32, + /// `get$name`. + getter: u32, + /// `set$name`, for the write-back. + setter: u32, + pos: rhai::Position, + }, + + /// `.name(args)`, with the receiver as the first argument by reference. + Method { + name: u32, + /// How many arguments, not counting the receiver. + argc: u8, + /// Where the first of them sits on the operand stack. + operand: u16, + pos: rhai::Position, + }, +} + +impl Step { + /// Where this step is in the source. + /// + /// Every step carries one, and it is the one place diagnostics are not + /// strippable — four bytes per step, in the chain pool rather than the + /// position table. That is not an oversight twice over: a chain is a single + /// instruction, so the one entry the table holds for it cannot say which of + /// `a.b[i].c()` failed, and rhai blames the step rather than the chain for + /// all three kinds. An index is reported against its index expression, a + /// property against the property (`eval/chaining.rs:1039`), a method + /// against the call (`:904`). + #[must_use] + pub fn pos(&self) -> rhai::Position { + match self { + Step::Index { pos, .. } | Step::Property { pos, .. } | Step::Method { pos, .. } => *pos, + } + } +} + +/// What a chain does when it gets to the end. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Tail { + /// Push the value the chain arrived at. + Read, + /// Assign the top of the operand stack to it, optionally through an + /// operator, and push unit. + Assign { + /// Index into the op-assignment pool; absent for a plain `=`. + op: Option, + }, +} + +/// Where a chain starts. +/// +/// The distinction is whether the root has an identity to write back into. +/// Rhai draws the same line and in the same place: a variable root becomes a +/// `Target` into the scope entry, and anything else is evaluated into a +/// temporary and walked there (`eval/chaining.rs:547-571`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Root { + /// A local, by slot. + /// + /// The only root a chain can write *through*: a mutation partway down has + /// to land in the scope entry, and walking a copy of it would lose the + /// mutation. + Local { + slot: u16, + /// Names it in `ErrorAssignmentToConstant`. + name: u32, + }, + + /// A variable no slot addresses: one the caller put in the `Scope`, one a + /// resolver answers for, or a module's constant. + /// + /// Whether it can be written through is not known until it is looked up, + /// which is the whole difference from [`Root::Local`]. Rhai decides the + /// same way and at the same moment: `search_namespace` hands back a + /// `Target`, and a scope entry becomes a reference where a resolver's + /// answer or a module's constant becomes a read-only temporary + /// (`eval/expr.rs:120-155`). + /// + /// Carries its own position because the lookup can fail and + /// `ErrorVariableNotFound` is reported against the variable, not the + /// chain. That costs nothing extra: chain positions already live in this + /// pool rather than in the strippable table, for the reason [`Step::pos`] + /// gives. + Named { name: u32, pos: rhai::Position }, + + /// A value the instruction takes off the operand stack, pushed above the + /// step operands. + /// + /// `[1, 2, 3].len()`, `f().x`, `(a + b).to_string()`. Nothing is written + /// back, because there is nowhere to write it back to — and nothing can be + /// assigned to one, because rhai's parser refuses `f().x = 1` before this + /// ever sees it (`eval/chaining.rs:559`). + Temporary, +} + +/// A whole `a.b[i].c` chain. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Chain { + pub root: Root, + pub steps: Vec, + pub tail: Tail, + /// How many operand-stack values the *steps* consume, so the VM knows + /// where they start. Not the whole instruction's appetite — see + /// [`Chain::consumes`]. + pub operands: u16, +} + +impl Chain { + /// Whether the chain takes a value off the operand stack to store. + #[must_use] + pub fn assigns(&self) -> bool { + matches!(self.tail, Tail::Assign { .. }) + } + + /// Whether the root itself arrives on the operand stack. + /// + /// Only a temporary does. The other two are reached where they live — by + /// slot or by name — and getting this wrong is not a small mistake: + /// [`Chain::consumes`] is what the verifier models a chunk's whole operand + /// depth on, and what the VM finds its operands with. + #[must_use] + pub fn roots_on_stack(&self) -> bool { + match self.root { + Root::Local { .. } | Root::Named { .. } => false, + Root::Temporary => true, + } + } + + /// Everything the instruction takes off the operand stack. + /// + /// Pushed in that order — step operands, then the root, then the value + /// being assigned — which is rhai's evaluation order and not the reading + /// order: it collects a chain's indices and arguments *before* it evaluates + /// what they are being applied to (`eval/chaining.rs:498-524` then `:562`). + #[must_use] + pub fn consumes(&self) -> usize { + self.operands as usize + usize::from(self.roots_on_stack()) + usize::from(self.assigns()) + } + + /// Whether walking this chain can change what it walks over. + /// + /// A read-only chain needs no write-back at all, which is worth knowing: + /// write-back on a temporary calls a setter, and calling one where rhai + /// would not is an observable difference on a host type. + #[must_use] + pub fn mutates(&self) -> bool { + matches!(self.tail, Tail::Assign { .. }) + || self + .steps + .iter() + .any(|step| matches!(step, Step::Method { .. })) + } +} diff --git a/src/grain/bytecode/chunk.rs b/src/grain/bytecode/chunk.rs new file mode 100644 index 000000000..e121bf9d0 --- /dev/null +++ b/src/grain/bytecode/chunk.rs @@ -0,0 +1,68 @@ +use crate::grain::bytecode::code::disassemble; +use crate::grain::bytecode::Op; + +/// One body of code: the top-level program, or one script function. +/// +/// Metadata only. Every chunk in a program shares a single instruction buffer, +/// concatenated in order, and a chunk names its span of it. That keeps one +/// position table and one instruction address across the whole program, so a +/// device reporting where it failed reports one number. +/// +/// `max_stack` exists so the VM can size its operand stack rather than growing +/// it. The compiler emits an upper bound it can compute without a depth walk — +/// one per instruction — and then replaces it with the high water the verifier +/// actually measured. On a device that difference is the whole reservation: a +/// chunk of 25 instructions rarely stacks more than three values. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Chunk { + entry: u32, + end: u32, + max_stack: u16, +} + +impl Chunk { + pub(crate) fn new(entry: u32, end: u32, max_stack: u16) -> Self { + Self { + entry, + end, + max_stack, + } + } + + /// Where execution starts, as an offset into the program's code. + #[must_use] + pub fn entry(&self) -> u32 { + self.entry + } + + /// One past the last byte of this chunk. + #[must_use] + pub fn end(&self) -> u32 { + self.end + } + + #[must_use] + pub fn max_stack(&self) -> u16 { + self.max_stack + } + + pub(crate) fn set_max_stack(&mut self, max_stack: u16) { + self.max_stack = max_stack; + } + + /// This chunk's slice of a program's code. + #[must_use] + pub fn body<'c>(&self, code: &'c [u8]) -> &'c [u8] { + code.get(self.entry as usize..self.end as usize) + .unwrap_or_default() + } + + /// The instructions, paired with their addresses in the program. + /// + /// For reading, not for running — reconstructing an [`Op`] is exactly the + /// work the byte encoding exists to avoid. + pub fn ops<'c>(&self, code: &'c [u8]) -> impl Iterator + 'c { + let entry = self.entry as usize; + disassemble(self.body(code)).map(move |(at, op)| (at + entry, op)) + } +} diff --git a/src/grain/bytecode/code.rs b/src/grain/bytecode/code.rs new file mode 100644 index 000000000..69622e938 --- /dev/null +++ b/src/grain/bytecode/code.rs @@ -0,0 +1,929 @@ +//! The executable form: instructions as bytes, run in place. +//! +//! [`Op`] is what the compiler emits and what a disassembly shows. It is not +//! what runs. A `Vec` costs sixteen bytes an instruction and has to be +//! built at load, which is most of what this project exists to avoid — so a +//! program's code is a byte slice, and a loaded program borrows it from the +//! artifact rather than decoding it into anything. +//! +//! ## Why fixed fields rather than varints +//! +//! Everything else in an artifact is LEB128, because everything else is read +//! once. Instructions are read on every execution, so their operands are +//! fixed-width little-endian at known offsets: decoding is a match on the tag +//! and a couple of loads, with no loop per field. That costs about one byte per +//! operand against the varint form and buys a dispatch loop that does not +//! decode. +//! +//! ## Jumps are byte offsets +//! +//! Instructions vary in length, so a jump names a byte offset rather than an +//! instruction index. [`assemble`] resolves the compiler's indices into offsets +//! once, and [`verify`](super::verify) proves every one of them lands on an +//! instruction boundary — without which a jump into the middle of an operand +//! would decode whatever the operand's bytes happen to look like. + +use alloc::borrow::Cow; + +use crate::grain::bytecode::{Op, Receiver}; + +/// Instruction tags. +/// +/// Written out rather than derived from [`Op`]'s order, so reordering the enum +/// for readability cannot change what a program means. Append only. +/// +/// Operators get their own tag rather than an optional field, so the common +/// call pays nothing for the one that carries a token. +pub mod tag { + pub const CONST: u8 = 0x01; + pub const UNIT: u8 = 0x02; + pub const FALSE: u8 = 0x03; + pub const TRUE: u8 = 0x04; + pub const LOAD_LOCAL: u8 = 0x05; + pub const STORE_LOCAL: u8 = 0x06; + pub const ASSIGN_LOCAL: u8 = 0x07; + pub const ASSIGN_LOCAL_OP: u8 = 0x08; + pub const DECLARE_LOCAL: u8 = 0x09; + pub const DECLARE_CONST: u8 = 0x0a; + pub const POP: u8 = 0x0b; + pub const JUMP: u8 = 0x0c; + pub const JUMP_IF_TRUE: u8 = 0x0d; + pub const JUMP_IF_FALSE: u8 = 0x0e; + pub const CALL: u8 = 0x0f; + pub const CALL_OP: u8 = 0x10; + pub const UNWIND_TO: u8 = 0x11; + pub const TICK: u8 = 0x12; + pub const RETURN: u8 = 0x13; + pub const EVAL_AST: u8 = 0x14; + pub const EVAL_AST_KEEP: u8 = 0x15; + pub const CHAIN: u8 = 0x16; + pub const MAKE_ARRAY: u8 = 0x17; + pub const SWITCH: u8 = 0x18; + pub const LOAD_NAMED: u8 = 0x19; + pub const ASSIGN_NAMED: u8 = 0x1a; + pub const ASSIGN_NAMED_OP: u8 = 0x1b; + pub const THROW: u8 = 0x1c; + pub const ITER_INIT: u8 = 0x1d; + pub const ITER_NEXT: u8 = 0x1e; + pub const ITER_DROP: u8 = 0x1f; + pub const STORE_SHARED: u8 = 0x20; + pub const ITER_NEXT_INDEXED: u8 = 0x21; + pub const POP_HANDLER: u8 = 0x22; + pub const PUSH_HANDLER: u8 = 0x23; + pub const PUSH_HANDLER_VAR: u8 = 0x24; + pub const INTERPOLATE_START: u8 = 0x25; + pub const INTERPOLATE_APPEND: u8 = 0x26; + pub const INTERPOLATE_END: u8 = 0x27; + pub const MAKE_FN_PTR: u8 = 0x28; + pub const CURRY: u8 = 0x29; + pub const CALL_FN_PTR: u8 = 0x2a; + pub const CALL_FN_PTR_METHOD: u8 = 0x2b; + pub const SHARE: u8 = 0x2c; + pub const SHARE_NAMED: u8 = 0x2d; + pub const LOAD_SHARED: u8 = 0x2e; + pub const MAKE_CLOSURE: u8 = 0x2f; + pub const IS_SHARED: u8 = 0x30; + pub const CHECKPOINT: u8 = 0x31; + pub const CHECK_ARRAY_SIZE: u8 = 0x32; + pub const CHECK_MAP_SIZE: u8 = 0x33; + pub const MAKE_MAP: u8 = 0x34; + pub const CALL_LOCAL_REF: u8 = 0x35; + pub const ROTATE: u8 = 0x36; + pub const CALL_NAMED_REF: u8 = 0x37; + pub const LOAD_SHARED_NAMED: u8 = 0x38; +} + +/// How wide each tag's instruction is, with 0 for the tags that are not one. +/// +/// A table rather than a match because the dispatch loop needs the width of +/// every instruction it executes: matching on the tag twice, once to advance +/// and once to act, is a branch per instruction bought for nothing. +static WIDTHS: [u8; 256] = { + let mut widths = [0u8; 256]; + + widths[tag::UNIT as usize] = 1; + widths[tag::FALSE as usize] = 1; + widths[tag::TRUE as usize] = 1; + widths[tag::POP as usize] = 1; + widths[tag::TICK as usize] = 1; + widths[tag::CHECKPOINT as usize] = 1; + widths[tag::MAKE_MAP as usize] = 3; + widths[tag::CHECK_ARRAY_SIZE as usize] = 3; + widths[tag::CHECK_MAP_SIZE as usize] = 3; + widths[tag::RETURN as usize] = 1; + widths[tag::THROW as usize] = 1; + widths[tag::ITER_INIT as usize] = 1; + widths[tag::ITER_DROP as usize] = 1; + + widths[tag::STORE_SHARED as usize] = 3; + + widths[tag::ITER_NEXT as usize] = 5; + widths[tag::ITER_NEXT_INDEXED as usize] = 5; + widths[tag::POP_HANDLER as usize] = 1; + widths[tag::INTERPOLATE_START as usize] = 1; + widths[tag::INTERPOLATE_APPEND as usize] = 1; + widths[tag::INTERPOLATE_END as usize] = 1; + widths[tag::MAKE_FN_PTR as usize] = 1; + widths[tag::IS_SHARED as usize] = 1; + + widths[tag::CURRY as usize] = 2; + widths[tag::ROTATE as usize] = 2; + widths[tag::CALL_FN_PTR as usize] = 2; + widths[tag::CALL_FN_PTR_METHOD as usize] = 2; + + widths[tag::SHARE as usize] = 3; + widths[tag::SHARE_NAMED as usize] = 3; + widths[tag::LOAD_SHARED as usize] = 3; + widths[tag::LOAD_SHARED_NAMED as usize] = 3; + widths[tag::MAKE_CLOSURE as usize] = 3; + widths[tag::PUSH_HANDLER as usize] = 5; + widths[tag::PUSH_HANDLER_VAR as usize] = 7; + + widths[tag::CONST as usize] = 3; + widths[tag::LOAD_LOCAL as usize] = 3; + widths[tag::STORE_LOCAL as usize] = 3; + widths[tag::DECLARE_LOCAL as usize] = 3; + widths[tag::DECLARE_CONST as usize] = 3; + widths[tag::UNWIND_TO as usize] = 3; + widths[tag::EVAL_AST as usize] = 3; + widths[tag::EVAL_AST_KEEP as usize] = 3; + widths[tag::CHAIN as usize] = 3; + widths[tag::MAKE_ARRAY as usize] = 3; + widths[tag::SWITCH as usize] = 3; + widths[tag::LOAD_NAMED as usize] = 3; + widths[tag::ASSIGN_NAMED as usize] = 3; + + widths[tag::ASSIGN_NAMED_OP as usize] = 5; + + widths[tag::CALL as usize] = 4; + + widths[tag::ASSIGN_LOCAL as usize] = 5; + widths[tag::JUMP as usize] = 5; + widths[tag::JUMP_IF_TRUE as usize] = 5; + widths[tag::JUMP_IF_FALSE as usize] = 5; + + widths[tag::CALL_OP as usize] = 6; + widths[tag::CALL_LOCAL_REF as usize] = 6; + widths[tag::CALL_NAMED_REF as usize] = 6; + + widths[tag::ASSIGN_LOCAL_OP as usize] = 7; + + widths +}; + +/// How many bytes the instruction at `at` occupies, or `None` if the tag is +/// unknown or the operands run past the end. +#[must_use] +#[inline] +pub fn width(code: &[u8], at: usize) -> Option { + let size = WIDTHS[*code.get(at)? as usize] as usize; + if size == 0 { + return None; + } + // An instruction whose operands are cut off is not an instruction. + (at + size <= code.len()).then_some(size) +} + +/// Read a `u16` operand at `at`. +/// +/// Returns `None` past the end rather than panicking. The verifier makes that +/// unreachable for any program the VM will run, but the check is a load's worth +/// of cost and it means nothing has to be trusted. +#[must_use] +#[inline] +pub fn u16_at(code: &[u8], at: usize) -> Option { + Some(u16::from_le_bytes(code.get(at..at + 2)?.try_into().ok()?)) +} + +/// Read a `u32` operand at `at`. +#[must_use] +#[inline] +pub fn u32_at(code: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(code.get(at..at + 4)?.try_into().ok()?)) +} + +/// Why a lowering could not be turned into bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AssembleError { + /// A pool grew past what a `u16` operand can name. The compiler falls back + /// to a whole-program fragment rather than emitting a truncated index. + PoolTooLarge { what: &'static str, entries: usize }, + /// A jump naming an instruction that does not exist. A compiler bug. + JumpOutOfRange { at: usize, target: u32 }, + /// The same, for a jump that lives in a switch table rather than in the + /// code. + SwitchTargetOutOfRange { table: usize, target: u32 }, + /// A chunk longer than a `u32` of bytes. + ChunkTooLarge { bytes: usize }, +} + +/// Pack instructions into their executable form. +/// +/// Two passes: the first measures each instruction so jump targets can be +/// turned from indices into byte offsets, the second writes. Also returns the +/// byte offset of each instruction, so the position table can be re-keyed from +/// indices onto addresses. +/// +/// # Errors +/// +/// [`AssembleError`] for anything that cannot be expressed in the operand +/// widths. All of them are compiler bugs except [`AssembleError::PoolTooLarge`], +/// which a large enough script can reach. +pub fn assemble(ops: &[Op]) -> Result<(Vec, Vec), AssembleError> { + let mut offsets = Vec::with_capacity(ops.len() + 1); + let mut at = 0u32; + for op in ops { + offsets.push(at); + at = at + .checked_add(encoded_width(op) as u32) + .ok_or(AssembleError::ChunkTooLarge { bytes: usize::MAX })?; + } + // One past the end, so a jump to "after the last instruction" resolves. + offsets.push(at); + + let mut code = Vec::with_capacity(at as usize); + for (index, op) in ops.iter().enumerate() { + let target = |target: u32| -> Result { + offsets + .get(target as usize) + .copied() + .ok_or(AssembleError::JumpOutOfRange { at: index, target }) + }; + + let small = |value: usize, what: &'static str| -> Result { + u16::try_from(value).map_err(|_| AssembleError::PoolTooLarge { + what, + entries: value, + }) + }; + + match op { + Op::Const(index) => { + code.push(tag::CONST); + code.extend_from_slice(&small(*index as usize, "constants")?.to_le_bytes()); + } + Op::Unit => code.push(tag::UNIT), + Op::Bool(false) => code.push(tag::FALSE), + Op::Bool(true) => code.push(tag::TRUE), + + Op::LoadLocal(slot) => { + code.push(tag::LOAD_LOCAL); + code.extend_from_slice(&slot.to_le_bytes()); + } + Op::StoreLocal(slot) => { + code.push(tag::STORE_LOCAL); + code.extend_from_slice(&slot.to_le_bytes()); + } + + Op::AssignLocal { + slot, + var_name, + op: None, + } => { + code.push(tag::ASSIGN_LOCAL); + code.extend_from_slice(&slot.to_le_bytes()); + code.extend_from_slice(&small(*var_name as usize, "names")?.to_le_bytes()); + } + Op::AssignLocal { + slot, + var_name, + op: Some(assign_op), + } => { + code.push(tag::ASSIGN_LOCAL_OP); + code.extend_from_slice(&slot.to_le_bytes()); + code.extend_from_slice(&small(*var_name as usize, "names")?.to_le_bytes()); + code.extend_from_slice( + &small(*assign_op as usize, "op-assignments")?.to_le_bytes(), + ); + } + + Op::LoadNamed(name) => { + code.push(tag::LOAD_NAMED); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + + Op::AssignNamed { name, op: None } => { + code.push(tag::ASSIGN_NAMED); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + Op::AssignNamed { + name, + op: Some(assign_op), + } => { + code.push(tag::ASSIGN_NAMED_OP); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + code.extend_from_slice( + &small(*assign_op as usize, "op-assignments")?.to_le_bytes(), + ); + } + + Op::DeclareLocal { name, is_const } => { + code.push(if *is_const { + tag::DECLARE_CONST + } else { + tag::DECLARE_LOCAL + }); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + + Op::Pop => code.push(tag::POP), + + Op::Jump(to) => { + code.push(tag::JUMP); + code.extend_from_slice(&target(*to)?.to_le_bytes()); + } + Op::JumpIfTrue { target: to } => { + code.push(tag::JUMP_IF_TRUE); + code.extend_from_slice(&target(*to)?.to_le_bytes()); + } + Op::JumpIfFalse { target: to } => { + code.push(tag::JUMP_IF_FALSE); + code.extend_from_slice(&target(*to)?.to_le_bytes()); + } + + Op::Call { + name, + argc, + op: None, + } => { + code.push(tag::CALL); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + code.push(*argc); + } + Op::Call { + name, + argc, + op: Some(token), + } => { + code.push(tag::CALL_OP); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + code.push(*argc); + code.extend_from_slice(&small(*token as usize, "operators")?.to_le_bytes()); + } + + Op::CallRef { + name, + argc, + receiver, + } => { + let (tag, operand) = match receiver { + Receiver::Local(slot) => (tag::CALL_LOCAL_REF, *slot), + Receiver::Named(var) => (tag::CALL_NAMED_REF, small(*var as usize, "names")?), + }; + code.push(tag); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + code.push(*argc); + code.extend_from_slice(&operand.to_le_bytes()); + } + + Op::Rotate(under) => { + code.push(tag::ROTATE); + code.push(*under); + } + + Op::Chain(index) => { + code.push(tag::CHAIN); + code.extend_from_slice(&small(*index as usize, "chains")?.to_le_bytes()); + } + + Op::MakeArray(len) => { + code.push(tag::MAKE_ARRAY); + code.extend_from_slice(&len.to_le_bytes()); + } + + Op::MakeMap(len) => { + code.push(tag::MAKE_MAP); + code.extend_from_slice(&len.to_le_bytes()); + } + + Op::CheckSize { index, map } => { + code.push(if *map { + tag::CHECK_MAP_SIZE + } else { + tag::CHECK_ARRAY_SIZE + }); + code.extend_from_slice(&index.to_le_bytes()); + } + + Op::Share(slot) => { + code.push(tag::SHARE); + code.extend_from_slice(&slot.to_le_bytes()); + } + Op::ShareNamed(name) => { + code.push(tag::SHARE_NAMED); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + Op::LoadShared(slot) => { + code.push(tag::LOAD_SHARED); + code.extend_from_slice(&slot.to_le_bytes()); + } + Op::LoadSharedNamed(name) => { + code.push(tag::LOAD_SHARED_NAMED); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + + Op::MakeClosure(name) => { + code.push(tag::MAKE_CLOSURE); + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + Op::MakeFnPtr => code.push(tag::MAKE_FN_PTR), + Op::IsShared => code.push(tag::IS_SHARED), + Op::Curry(argc) => { + code.push(tag::CURRY); + code.push(*argc); + } + Op::CallFnPtr { argc, method } => { + code.push(if *method { + tag::CALL_FN_PTR_METHOD + } else { + tag::CALL_FN_PTR + }); + code.push(*argc); + } + + Op::InterpolateStart => code.push(tag::INTERPOLATE_START), + Op::InterpolateAppend => code.push(tag::INTERPOLATE_APPEND), + Op::InterpolateEnd => code.push(tag::INTERPOLATE_END), + + // The table's own targets are instruction indices too, but they + // are not in the code — see `resolve_switch_targets`. + Op::Switch(index) => { + code.push(tag::SWITCH); + code.extend_from_slice(&small(*index as usize, "switches")?.to_le_bytes()); + } + + Op::UnwindTo(depth) => { + code.push(tag::UNWIND_TO); + code.extend_from_slice(&depth.to_le_bytes()); + } + + Op::Tick => code.push(tag::TICK), + Op::Checkpoint => code.push(tag::CHECKPOINT), + Op::Throw => code.push(tag::THROW), + Op::IterInit => code.push(tag::ITER_INIT), + Op::IterDrop => code.push(tag::ITER_DROP), + Op::PopHandler => code.push(tag::POP_HANDLER), + + Op::PushHandler { + target: to, + catch_var, + } => { + code.push(match catch_var { + Some(..) => tag::PUSH_HANDLER_VAR, + None => tag::PUSH_HANDLER, + }); + code.extend_from_slice(&target(*to)?.to_le_bytes()); + if let Some(name) = catch_var { + code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); + } + } + + Op::IterNext { exit, indexed } => { + code.push(if *indexed { + tag::ITER_NEXT_INDEXED + } else { + tag::ITER_NEXT + }); + code.extend_from_slice(&target(*exit)?.to_le_bytes()); + } + + Op::StoreShared(slot) => { + code.push(tag::STORE_SHARED); + code.extend_from_slice(&slot.to_le_bytes()); + } + Op::Return => code.push(tag::RETURN), + + Op::EvalAst { + residual, + rewind_scope, + } => { + code.push(if *rewind_scope { + tag::EVAL_AST + } else { + tag::EVAL_AST_KEEP + }); + code.extend_from_slice(&small(*residual as usize, "fragments")?.to_le_bytes()); + } + } + } + + Ok((code, offsets)) +} + +/// Rewrite switch targets from instruction indices into byte offsets. +/// +/// The other half of [`assemble`], and separate only because a table is not in +/// the instruction stream: [`Op::Switch`] carries a pool index, and the jumps +/// are inside the pool entry. Same `offsets` table, same one-past-the-end +/// entry, so a `switch` whose default is the end of the chunk resolves. +/// +/// # Errors +/// +/// [`AssembleError::SwitchTargetOutOfRange`] for a target naming no +/// instruction, which is a compiler bug. +pub fn resolve_switch_targets( + switches: &mut [super::Switch], + offsets: &[u32], +) -> Result<(), AssembleError> { + for (table, switch) in switches.iter_mut().enumerate() { + let resolve = |target: &mut u32| -> Result<(), AssembleError> { + *target = + *offsets + .get(*target as usize) + .ok_or(AssembleError::SwitchTargetOutOfRange { + table, + target: *target, + })?; + Ok(()) + }; + + for case in &mut switch.cases { + resolve(&mut case.target)?; + } + for range in &mut switch.ranges { + resolve(&mut range.target)?; + } + resolve(&mut switch.default)?; + } + Ok(()) +} + +/// How many bytes an instruction will take once written. +fn encoded_width(op: &Op) -> usize { + match op { + Op::Unit + | Op::Bool(..) + | Op::Pop + | Op::Tick + | Op::Checkpoint + | Op::Throw + | Op::IterInit + | Op::IterDrop + | Op::PopHandler + | Op::InterpolateStart + | Op::InterpolateAppend + | Op::InterpolateEnd + | Op::MakeFnPtr + | Op::IsShared + | Op::Return => 1, + Op::Curry(..) | Op::CallFnPtr { .. } | Op::Rotate(..) => 2, + Op::Const(..) + | Op::LoadLocal(..) + | Op::StoreLocal(..) + | Op::DeclareLocal { .. } + | Op::UnwindTo(..) + | Op::EvalAst { .. } + | Op::Chain(..) + | Op::Switch(..) + | Op::LoadNamed(..) + | Op::AssignNamed { op: None, .. } + | Op::StoreShared(..) + | Op::Share(..) + | Op::ShareNamed(..) + | Op::LoadShared(..) + | Op::LoadSharedNamed(..) + | Op::MakeClosure(..) + | Op::MakeArray(..) + | Op::MakeMap(..) + | Op::CheckSize { .. } => 3, + Op::Call { op: None, .. } => 4, + Op::AssignLocal { op: None, .. } + | Op::AssignNamed { op: Some(..), .. } + | Op::Jump(..) + | Op::JumpIfTrue { .. } + | Op::JumpIfFalse { .. } + | Op::IterNext { .. } + | Op::PushHandler { + catch_var: None, .. + } => 5, + Op::PushHandler { + catch_var: Some(..), + .. + } => 7, + Op::Call { op: Some(..), .. } | Op::CallRef { .. } => 6, + Op::AssignLocal { op: Some(..), .. } => 7, + } +} + +/// Recover the instruction at `at`, for disassembly and tests. +/// +/// Jump targets come back as byte offsets, not the instruction indices the +/// compiler used, because that is what the code actually holds. +#[must_use] +pub fn decode(code: &[u8], at: usize) -> Option { + width(code, at)?; + let small = |offset: usize| u16_at(code, at + offset); + + Some(match code[at] { + tag::CONST => Op::Const(u32::from(small(1)?)), + tag::UNIT => Op::Unit, + tag::FALSE => Op::Bool(false), + tag::TRUE => Op::Bool(true), + + tag::LOAD_LOCAL => Op::LoadLocal(small(1)?), + tag::STORE_LOCAL => Op::StoreLocal(small(1)?), + + tag::ASSIGN_LOCAL => Op::AssignLocal { + slot: small(1)?, + var_name: u32::from(small(3)?), + op: None, + }, + tag::ASSIGN_LOCAL_OP => Op::AssignLocal { + slot: small(1)?, + var_name: u32::from(small(3)?), + op: Some(u32::from(small(5)?)), + }, + + tag::LOAD_NAMED => Op::LoadNamed(u32::from(small(1)?)), + tag::ASSIGN_NAMED => Op::AssignNamed { + name: u32::from(small(1)?), + op: None, + }, + tag::ASSIGN_NAMED_OP => Op::AssignNamed { + name: u32::from(small(1)?), + op: Some(u32::from(small(3)?)), + }, + + tag::DECLARE_LOCAL => Op::DeclareLocal { + name: u32::from(small(1)?), + is_const: false, + }, + tag::DECLARE_CONST => Op::DeclareLocal { + name: u32::from(small(1)?), + is_const: true, + }, + + tag::POP => Op::Pop, + + tag::JUMP => Op::Jump(u32_at(code, at + 1)?), + tag::JUMP_IF_TRUE => Op::JumpIfTrue { + target: u32_at(code, at + 1)?, + }, + tag::JUMP_IF_FALSE => Op::JumpIfFalse { + target: u32_at(code, at + 1)?, + }, + + tag::CALL => Op::Call { + name: u32::from(small(1)?), + argc: code[at + 3], + op: None, + }, + tag::CALL_OP => Op::Call { + name: u32::from(small(1)?), + argc: code[at + 3], + op: Some(u32::from(small(4)?)), + }, + + tag::CALL_LOCAL_REF => Op::CallRef { + name: u32::from(small(1)?), + argc: code[at + 3], + receiver: Receiver::Local(small(4)?), + }, + tag::CALL_NAMED_REF => Op::CallRef { + name: u32::from(small(1)?), + argc: code[at + 3], + receiver: Receiver::Named(u32::from(small(4)?)), + }, + tag::ROTATE => Op::Rotate(code[at + 1]), + + tag::CHAIN => Op::Chain(u32::from(small(1)?)), + tag::SWITCH => Op::Switch(u32::from(small(1)?)), + tag::MAKE_ARRAY => Op::MakeArray(small(1)?), + tag::MAKE_MAP => Op::MakeMap(small(1)?), + tag::CHECK_ARRAY_SIZE => Op::CheckSize { + index: small(1)?, + map: false, + }, + tag::CHECK_MAP_SIZE => Op::CheckSize { + index: small(1)?, + map: true, + }, + tag::SHARE => Op::Share(small(1)?), + tag::SHARE_NAMED => Op::ShareNamed(u32::from(small(1)?)), + tag::LOAD_SHARED => Op::LoadShared(small(1)?), + tag::LOAD_SHARED_NAMED => Op::LoadSharedNamed(u32::from(small(1)?)), + tag::MAKE_CLOSURE => Op::MakeClosure(u32::from(small(1)?)), + tag::MAKE_FN_PTR => Op::MakeFnPtr, + tag::IS_SHARED => Op::IsShared, + tag::CURRY => Op::Curry(code[at + 1]), + tag::CALL_FN_PTR => Op::CallFnPtr { + argc: code[at + 1], + method: false, + }, + tag::CALL_FN_PTR_METHOD => Op::CallFnPtr { + argc: code[at + 1], + method: true, + }, + tag::INTERPOLATE_START => Op::InterpolateStart, + tag::INTERPOLATE_APPEND => Op::InterpolateAppend, + tag::INTERPOLATE_END => Op::InterpolateEnd, + tag::UNWIND_TO => Op::UnwindTo(small(1)?), + tag::TICK => Op::Tick, + tag::CHECKPOINT => Op::Checkpoint, + tag::THROW => Op::Throw, + tag::ITER_INIT => Op::IterInit, + tag::ITER_DROP => Op::IterDrop, + tag::ITER_NEXT => Op::IterNext { + exit: u32_at(code, at + 1)?, + indexed: false, + }, + tag::ITER_NEXT_INDEXED => Op::IterNext { + exit: u32_at(code, at + 1)?, + indexed: true, + }, + tag::STORE_SHARED => Op::StoreShared(small(1)?), + tag::POP_HANDLER => Op::PopHandler, + tag::PUSH_HANDLER => Op::PushHandler { + target: u32_at(code, at + 1)?, + catch_var: None, + }, + tag::PUSH_HANDLER_VAR => Op::PushHandler { + target: u32_at(code, at + 1)?, + catch_var: Some(u32::from(small(5)?)), + }, + tag::RETURN => Op::Return, + + tag::EVAL_AST => Op::EvalAst { + residual: u32::from(small(1)?), + rewind_scope: true, + }, + tag::EVAL_AST_KEEP => Op::EvalAst { + residual: u32::from(small(1)?), + rewind_scope: false, + }, + + _ => return None, + }) +} + +/// Every instruction in a chunk, paired with its address. +/// +/// Stops at the first thing it cannot decode, so it is safe to point at +/// anything. For a chunk that verified, it reaches the end. +pub fn disassemble(code: &[u8]) -> impl Iterator + '_ { + let mut at = 0usize; + core::iter::from_fn(move || { + let op = decode(code, at)?; + let here = at; + at += width(code, at)?; + Some((here, op)) + }) +} + +/// A chunk's instructions, owned when compiled and borrowed when loaded. +/// +/// Borrowing is the point. A program read from an artifact holds a slice of +/// those bytes and allocates nothing for its code — which is the difference +/// between retaining sixteen bytes an instruction and retaining three. +pub type Code<'a> = Cow<'a, [u8]>; + +#[cfg(test)] +mod tests { + use super::*; + + /// The property everything else rests on: what the compiler emitted is what + /// comes back, with jumps rewritten from indices to the addresses those + /// instructions actually landed at. + #[test] + fn instructions_survive_assembly() { + let ops = vec![ + Op::Const(7), + Op::Unit, + Op::Bool(true), + Op::Bool(false), + Op::LoadLocal(3), + Op::StoreLocal(4), + Op::AssignLocal { + slot: 1, + var_name: 2, + op: None, + }, + Op::AssignLocal { + slot: 1, + var_name: 2, + op: Some(5), + }, + Op::DeclareLocal { + name: 8, + is_const: false, + }, + Op::DeclareLocal { + name: 9, + is_const: true, + }, + Op::Pop, + Op::Call { + name: 1, + argc: 2, + op: None, + }, + Op::Call { + name: 1, + argc: 2, + op: Some(3), + }, + Op::CallRef { + name: 1, + argc: 2, + receiver: Receiver::Local(4), + }, + Op::CallRef { + name: 1, + argc: 2, + receiver: Receiver::Named(5), + }, + Op::Rotate(3), + Op::UnwindTo(6), + Op::Tick, + Op::EvalAst { + residual: 0, + rewind_scope: true, + }, + Op::EvalAst { + residual: 1, + rewind_scope: false, + }, + Op::Return, + ]; + + let (code, offsets) = assemble(&ops).expect("must assemble"); + let back: Vec<_> = disassemble(&code).map(|(_, op)| op).collect(); + + assert_eq!(back, ops); + assert_eq!(offsets.len(), ops.len() + 1); + assert_eq!( + *offsets.last().unwrap() as usize, + code.len(), + "the trailing offset must be the end of the chunk", + ); + } + + #[test] + fn jumps_become_the_addresses_of_the_instructions_they_named() { + // Index 3 is `Return`, which lands after Const(3) + Unit + Pop. + let ops = vec![Op::Const(0), Op::Unit, Op::Pop, Op::Return, Op::Jump(3)]; + let (code, offsets) = assemble(&ops).expect("must assemble"); + + assert_eq!(offsets[3], 3 + 1 + 1); + assert_eq!(decode(&code, offsets[4] as usize), Some(Op::Jump(5))); + } + + /// The compiler emits a jump to "one past the last instruction" when a + /// block's exit is the end of the chunk. + #[test] + fn a_jump_past_the_last_instruction_resolves_to_the_end() { + let ops = vec![Op::Unit, Op::Jump(2)]; + let (code, _) = assemble(&ops).expect("must assemble"); + assert_eq!(decode(&code, 1), Some(Op::Jump(code.len() as u32))); + } + + #[test] + fn a_jump_to_an_instruction_that_does_not_exist_is_refused() { + assert_eq!( + assemble(&[Op::Jump(99)]), + Err(AssembleError::JumpOutOfRange { at: 0, target: 99 }), + ); + } + + /// Operand widths are the format's hard limit, and the compiler falls back + /// rather than writing a truncated index. + #[test] + fn a_pool_index_too_wide_for_its_operand_is_refused() { + assert_eq!( + assemble(&[Op::Const(70_000)]), + Err(AssembleError::PoolTooLarge { + what: "constants", + entries: 70_000, + }), + ); + } + + #[test] + fn an_unknown_tag_has_no_width_and_does_not_decode() { + assert_eq!(width(&[0xff, 0, 0], 0), None); + assert_eq!(decode(&[0xff, 0, 0], 0), None); + } + + /// A truncated operand must not read whatever follows in memory. + #[test] + fn an_instruction_cut_short_does_not_decode() { + assert_eq!( + width(&[tag::CONST, 0], 0), + None, + "one byte of a u16 operand" + ); + assert_eq!(decode(&[tag::CONST, 0], 0), None); + assert_eq!(decode(&[tag::JUMP, 0, 0], 0), None); + } + + /// Disassembly is pointed at untrusted bytes by `dump`, so it stops rather + /// than running away. + #[test] + fn disassembling_junk_terminates() { + let junk = [0xff; 32]; + assert_eq!(disassemble(&junk).count(), 0); + + let partly_good = [tag::UNIT, tag::POP, 0xff, tag::UNIT]; + assert_eq!(disassemble(&partly_good).count(), 2); + } +} diff --git a/src/grain/bytecode/mod.rs b/src/grain/bytecode/mod.rs new file mode 100644 index 000000000..88b176dc9 --- /dev/null +++ b/src/grain/bytecode/mod.rs @@ -0,0 +1,18 @@ +pub mod code; + +mod chain; +mod chunk; +mod strings; +mod switch; +mod op; +mod positions; +mod verify; + +pub use chain::{Chain, Root, Step, Tail}; +pub use strings::{BadTable, Strings}; +pub use switch::{probe, Switch, SwitchCase, SwitchRange}; +pub use chunk::Chunk; +pub use code::{assemble, disassemble, resolve_switch_targets, AssembleError, Code}; +pub use op::{AssignOp, Op, Receiver}; +pub use positions::{Positions, TableError}; +pub use verify::{verify, Pools, VerifyError}; diff --git a/src/grain/bytecode/op.rs b/src/grain/bytecode/op.rs new file mode 100644 index 000000000..83191b720 --- /dev/null +++ b/src/grain/bytecode/op.rs @@ -0,0 +1,496 @@ +use crate::tokenizer::Token; + +/// What `x op= y` needs to reproduce rhai's resolution order. +/// +/// Both the op-assignment and the plain operator are carried, because rhai +/// tries the first and falls back to expanding into the second when no +/// op-assignment implementation exists (`eval/stmt.rs:217-236`). +/// +/// Lives in the program's op-assignment pool rather than in the instruction: +/// four fields including two `Token`s do not fit an operand, and the same +/// `+=` used in ten places is one entry. +// No `Eq`: `Token` carries float literals, so it is only `PartialEq`. +#[derive(Debug, Clone, PartialEq)] +pub struct AssignOp { + /// The `+=` token, for the built-in lookup. + pub op_assign: Token, + /// `"+="`, for dispatch and for error messages. + pub op_assign_name: u32, + /// The `+` token, for the expansion. + pub op: Token, + /// `"+"`. + pub op_name: u32, +} + +/// Where [`Op::CallRef`] finds the variable it calls through. +/// +/// The two differ in how the variable is reached, not in what happens to it: +/// both take a reference where rhai would and fall back to a value where it +/// would not, by the same rule. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Receiver { + /// A local, addressed by slot. Nothing was pushed for it — the call reads + /// the scope entry itself, and a slot always names one. + Local(u16), + + /// A variable no slot addresses: the caller's, a module's, or nothing. + /// + /// [`Op::LoadNamed`] has already resolved the name and left its value as + /// argument zero, which is what raises `ErrorVariableNotFound` against the + /// variable rather than against the call — two positions the table cannot + /// give one instruction. The call re-reaches the scope entry for the + /// reference and falls back to that value when there is no entry to reach: + /// a resolver's answer, a module's constant, a `const`. + /// + /// So the by-reference path pays for a clone it discards. Worth removing + /// only if a profile of a host-heavy script says so; a local, which is the + /// common receiver by far, never makes one. + Named(u32), +} + +/// One VM instruction, as the compiler emits it and a disassembly shows it. +/// +/// **Not the executed form.** A program's code is a byte slice, assembled from +/// these by [`assemble`](crate::bytecode::assemble) and dispatched on directly, +/// so a loaded program can borrow its instructions from the artifact rather +/// than building sixteen bytes of enum per instruction. See +/// [`code`](crate::bytecode::code) for the encoding. +/// +/// A stack machine: operands are pushed and consumed on an operand stack, and +/// locals live in slots addressed directly. `EvalAst` is the escape hatch that +/// hands a fragment back to rhai's tree walker, so anything the compiler cannot +/// yet lower still runs, and the whole language stays covered. Milestones +/// convert residuals into instructions rather than adding coverage. +/// +/// Instructions carry no source position. Several of them can fail against a +/// place in the source, and the position for that comes from the program's +/// [`Positions`](crate::bytecode::Positions) table, keyed on the instruction's +/// own address. Keeping it out means the diagnostics can be stripped from an +/// artifact without touching the code. +/// +/// Anything too wide for an operand is a `u32` index into one of the program's +/// pools, which is also what keeps a repeated operator or name from being +/// stored twice. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Op { + /// Push constant `.0` from the pool. + Const(u32), + /// Push unit. + Unit, + /// Push a boolean. + Bool(bool), + + /// Push the value in local slot `.0`. + LoadLocal(u16), + /// Pop and write into local slot `.0`, which must already exist. + StoreLocal(u16), + + /// Push the value of the variable named `.0`, found by name. + /// + /// For the variables no slot can address: the ones the caller already had + /// in its `Scope` when the program started, which sit below the base every + /// slot is measured from. Without this, a script that reads anything its + /// host supplied is a fragment, and so cannot be written to an artifact at + /// all. + /// + /// Three places are searched, in rhai's order (`eval/expr.rs:107-155`): + /// the resolver a host may have registered with `Engine::on_var`, then the + /// scope, then the modules loaded into the global namespace. Missing from + /// all three is `ErrorVariableNotFound`. + /// + /// A reverse scan of the scope per read, where a slot is an index — which + /// is why the compiler only emits this for a name it could not resolve. + LoadNamed(u32), + + /// Pop a value and assign it to the variable named `name`, optionally + /// through an operator. + /// + /// [`Op::LoadNamed`]'s counterpart, and resolved the same way. Assigning + /// to anything that is not a scope entry it can take a reference to — a + /// value the resolver produced, a module's constant, a `const` — is + /// `ErrorAssignmentToConstant`, as it is in the walker. + AssignNamed { + name: u32, + /// Index into the op-assignment pool; absent for a plain `=`. + op: Option, + }, + + /// Pop a value and assign it to local slot `slot`, optionally through an + /// operator. + /// + /// Separate from `StoreLocal` because `x += y` is not `x = x + y`: rhai + /// looks for an op-assignment implementation that mutates in place, and + /// only expands to the binary form if there is none. + AssignLocal { + slot: u16, + /// Names the variable in `ErrorAssignmentToConstant`. + var_name: u32, + /// Index into the op-assignment pool; absent for a plain `=`. + op: Option, + }, + /// Pop and declare it as a new local, extending the scope by one. + /// + /// Slots are assigned in declaration order, so the new local always lands + /// at the top of the scope. Carries the name because locals live in the + /// caller's `Scope`, where entries are named, and carries constness + /// because rhai enforces it through the value's own access mode. + DeclareLocal { name: u32, is_const: bool }, + + /// Discard the top of the operand stack. + Pop, + + /// Jump to `.0`. + /// + /// An instruction index as the compiler emits it, a byte offset once + /// assembled — instructions vary in length, so there is nothing else it + /// could be. + Jump(u32), + /// Pop a condition and jump to `.0` if it is true. Mirrors + /// [`Op::JumpIfFalse`]; both exist so short-circuit `&&` and `||` lower + /// without an extra negation. + JumpIfTrue { target: u32 }, + /// Pop a condition and jump to `.0` if it is false. + /// + /// Its position-table entry is the condition's own position, because rhai + /// rejects a non-boolean guard against the guard expression rather than the + /// statement — and the differential harness compares error positions. + JumpIfFalse { target: u32 }, + + /// Pop `argc` arguments and call the function named by `name`, pushing the + /// result. + /// + /// Dispatch goes through rhai, so every registered function, operator and + /// script function resolves exactly as it would in the walker. Only calls + /// rhai handles syntactically before dispatch — `Fn`, `call`, `curry`, + /// `eval`, `is_def_var` — are excluded, and stay fragments. + /// + /// The position table's entry for this instruction is the call site. Rhai's + /// dispatch path takes one and reports failures against it; `call_fn_raw` + /// does not, so an error that comes back without a position gets this one. + /// + /// `op` indexes the operator pool when the call is an operator, and names + /// the token the built-in lookup keys on. The walker short-circuits these + /// to a function pointer rather than dispatching, and a VM that did not + /// would be slower than the tree it replaced. + Call { + name: u32, + argc: u8, + op: Option, + }, + + /// Call `name` with a variable as its first argument, taken by reference. + /// + /// Rhai rewrites `f(x, ..)` into `x.f(..)` whenever the first argument is a + /// plain variable, so that a `&mut` first parameter mutates the variable + /// rather than a copy (`func/call.rs:1434-1460`). `push(a, 2)` and + /// `a.push(2)` are the same call; only the second reached the mutation + /// through [`Op::Chain`]. + /// + /// Two things follow from the rewrite, and together they are why this is an + /// instruction rather than an argument order: + /// + /// * the variable is read *after* the other arguments, so an argument that + /// writes to it is seen; + /// * a shared or read-only variable is passed by value instead — rhai hands + /// out a reference to neither (`func/call.rs:1449-1454`). + /// + /// Operators never reach here: under `fast_operators` a binary one + /// short-circuits before the rewrite (`func/call.rs:1775`), so `a + b` + /// reads `a` first and needs no reference. + CallRef { + name: u32, + argc: u8, + receiver: Receiver, + }, + + /// Move the top of the operand stack down past `.0` values. + /// + /// A [`Receiver::Named`] receiver is resolved by [`Op::LoadNamed`] after + /// the other arguments, and this puts it back in argument order. + Rotate(u8), + + /// Pop a subject and jump to wherever switch table `.0` sends it. + /// + /// Always jumps — the table's default is where a subject that matches + /// nothing goes, and an absent `_` arm compiles to a jump past the + /// statement. Arms with guards are not table entries: the table sends a + /// subject to the head of a chain that tries each guard in source order + /// and falls through to the default, which is what keeps dispatch a + /// lookup. + /// + /// The table is in the program's switch pool rather than in the + /// instruction because it is unbounded, and because two arms of one + /// `switch` share it. + Switch(u32), + + /// Turn local slot `.0` into a shared cell, so a closure can capture it. + /// + /// Rhai's parser emits one of these per captured variable ahead of the + /// `curry` call that binds them (`parser.rs:3707`). Sharing is what makes + /// the closure and the enclosing scope see the same value afterwards; the + /// write-through in `place` is the other half. + /// + /// The variable resolver gets first refusal, as it does in + /// `eval/stmt.rs:998`: if a host's `on_var` answers the name, the variable + /// is *not* shared. + Share(u16), + + /// The same for a variable no slot names — one the caller supplied. + ShareNamed(u32), + + /// Push local slot `.0` without flattening it. + /// + /// A read normally hands back what a shared cell contains, which is right + /// for a value and wrong for a capture: currying a closure has to bind the + /// *cell*, or the closure gets a copy and stops being a closure. + LoadShared(u16), + + /// The same for a variable no slot names — one the caller supplied. + /// + /// [`Op::LoadNamed`] flattens, and a closure capturing a caller's variable + /// through that read binds a copy: the aliasing is dead, and a write to the + /// variable afterwards is invisible to the closure. The slot case has had + /// [`Op::LoadShared`] since closures were lowered at all; this is the half + /// that was missing. + LoadSharedNamed(u32), + + /// Pop a value and push whether it is a shared cell. + /// + /// Rhai answers `is_shared` before dispatch and registers no function for + /// it (`func/call.rs:1240`), so there is nothing to call — and the value + /// has to arrive unflattened, or the answer is always false. + IsShared, + + /// Push a function pointer to the compiled function named `.0`. + /// + /// A closure's, whose name the parser makes up (`anon$…`) and which + /// [`Op::MakeFnPtr`] would refuse — rhai only builds pointers to names a + /// script could have written. The name is known here, so unlike + /// `MakeFnPtr` it needs no operand on the stack. + MakeClosure(u32), + + /// Pop a name and push a function pointer to it. + /// + /// Deliberately the *late-bound* kind, carrying a name and nothing else. + /// Rhai's other kind embeds a `ScriptFuncDef` — an AST body — which is + /// both unreachable from outside the crate and exactly the allocation this + /// project exists to remove. Building our own means a pointer resolves + /// through the compiled function table like any other call, and a program + /// holding one can still be written to an artifact. + /// + /// The cost is that a `Normal` pointer is late-bound where rhai's is + /// early-bound: redefining the function after taking a pointer to it is + /// visible here and not in the walker. + MakeFnPtr, + + /// Pop `.0` arguments and a function pointer, and push the pointer with + /// those arguments bound to the front of it. + Curry(u8), + + /// Pop `argc` arguments and a target, and call a function pointer. + /// + /// A compiled function of that name and arity is called directly, with the + /// curried arguments spliced in front. Anything else — a native function, + /// a name that resolves elsewhere — goes to rhai's own `call_raw`. + /// + /// `method` distinguishes `f.call(x)` from `call(f, x)`, which are not the + /// same call. In method position a target that is *not* a pointer is not + /// an error: rhai takes the first argument as the pointer and binds the + /// target as `this` (`func/call.rs:816-919`), which is how a closure is + /// called against a receiver. + CallFnPtr { argc: u8, method: bool }, + + /// Push an empty buffer for an interpolated string to be built in. + /// + /// Interpolation is three instructions rather than one because rhai checks + /// the size limit after **every** segment and blames the segment that went + /// over. One instruction has one position-table entry, so it could not say + /// which; a pool of per-segment positions would say it but would not be + /// strippable, and diagnostics staying separable is the point of the + /// table. An instruction per segment puts each position exactly where the + /// rest of them live. + /// + /// The buffer is an ordinary operand, so a nested interpolation needs + /// nothing special. + InterpolateStart, + + /// Pop a segment and append it to the buffer beneath it. + /// + /// Not `+`, which is what it looks like: `+` is overridable and + /// interpolation is not, and the string-plus-anything operator skips this + /// size check. A string segment is written straight out and never reaches + /// dispatch; anything else goes through rhai's `to_string` rendering, + /// which consults native functions only. + InterpolateAppend, + + /// Replace the buffer with the interned string it built. + InterpolateEnd, + + /// Pop `.0` values and push them as an array. + /// + /// Only for a literal whose elements are not all constant — one that is + /// gets folded into the pool by rhai's own optimizer before this sees it. + MakeArray(u16), + + /// Build a map from a template and `n` key/value pairs above it. + /// + /// The stack holds `[template, k0, v0, .., k(n-1), v(n-1)]`. The template + /// is a constant map that already carries every key the literal mentions, + /// with the computed ones holding a placeholder — that is rhai's own shape + /// (`ast/expr.rs:283`), and it is why an entirely constant map never + /// reaches here: the optimizer has already folded it into the template + /// alone. + /// + /// Keys ride on the operand stack as string constants rather than in a + /// pool of their own. They are constants either way, and the constant pool + /// already deduplicates them across the program. + MakeMap(u16), + + /// Measure the value on top of the stack into the array literal being + /// built, and raise `ErrorDataTooLarge` if the running total is over. + /// + /// The operand is the element's index within its literal: zero starts a + /// fresh total, and [`Op::MakeArray`] discards it. That is what keeps + /// `[a, [b, c], d]` straight — the inner literal's total is pushed and + /// popped inside the outer one's. + /// + /// A separate instruction rather than work inside `MakeArray` because rhai + /// blames the *element* that tipped the total over + /// (`eval/expr.rs:328`), and one instruction has one position-table entry. + /// Putting it here rather than in a pool beside the element count is what + /// keeps those positions strippable, which matters more for a literal than + /// for a chain: an array can have any number of elements. + CheckSize { + index: u16, + /// Whether the element counts towards the map limit rather than the + /// array one. Rhai adds one to a different member of the triple for + /// each (`eval/expr.rs:323` against `:354`), so the same running total + /// cannot serve both. + map: bool, + }, + + /// Walk `a.b[i].c`, indexing the chain pool. + /// + /// One instruction for the whole chain rather than one per step, because + /// the walk holds a `&mut` into the container at every level and a borrow + /// cannot survive a trip round the dispatch loop. Index values and method + /// arguments were pushed before it, in step order. + /// + /// Pushes the value for a read, or unit for an assignment. + Chain(u32), + + /// Truncate the scope back to `.0` locals, dropping everything a block + /// declared. The compile-time slot model unwinds in step. + UnwindTo(u16), + + /// Count one operation against `max_operations`, and give `on_progress` a + /// chance to terminate. + /// + /// Emitted on loop back-edges. Rhai ticks per AST node, so counts differ; + /// what this preserves is that a limit is enforced and an interrupt is + /// honoured, which is what stops `loop {}` from being unkillable. + /// + /// Its table entry is read on every iteration rather than only on failure, + /// which is why the in-memory position table is dense. + Tick, + + /// Record the current scope length as the depth an error escaping this + /// chunk unwinds to. + /// + /// Rhai rewinds a nested block whether it is left normally or by a throw, + /// and never rewinds the top level of a chunk (`eval/stmt.rs`, and + /// `eval_global_statements` passing `rewind_scope = false`). The normal + /// path is [`Op::UnwindTo`], which an escaping error jumps straight past — + /// so the frame needs a floor to fall back to, and the last top-level + /// statement boundary is exactly it. + /// + /// Emitted once before each top-level statement of a chunk that runs in the + /// caller's scope, so it costs nothing per iteration and nothing at all to + /// a function body, whose scope is discarded whole. + Checkpoint, + + /// Evaluate residual AST fragment `residual` through rhai's walker, + /// pushing its value. + /// + /// `rewind_scope` reaches `eval_stmt_block` when the fragment is a block, + /// and decides whether locals it declares survive. Statement fragments + /// rewind, so they cannot disturb the scope shape slots were resolved + /// against. A whole-program fragment does not, because rhai does not + /// rewind top-level statements and callers can see what they declared. + EvalAst { residual: u32, rewind_scope: bool }, + + /// Arm a handler covering the instructions up to the matching + /// [`Op::PopHandler`], catching to `target`. + /// + /// Records where the operand stack, the scope and the iterator stack were + /// when it was armed, because an error can be raised at any depth of all + /// three and the catch block has to start where the `try` did. + /// + /// Only errors rhai considers catchable are caught: `return`, `break`, + /// `continue` and `exit` unwind as errors too and must pass straight + /// through (`eval/stmt.rs:806`). + /// + /// `catch_var` names the variable the error is bound to. Its table entry + /// is that variable's position, which is what rhai reports + /// `ErrorTooManyVariables` against. + PushHandler { target: u32, catch_var: Option }, + + /// Disarm the innermost handler. + /// + /// Emitted twice per `try`: once where the body ends normally, and once + /// where the catch block does — the second ends the region in which a + /// bare `throw;` means "re-raise the original". + PopHandler, + + /// Pop an iterable and start iterating it. + /// + /// The iterator goes on a stack of the VM's own rather than the operand + /// stack, because it is not a `Dynamic`. Rhai's iterator functions take + /// the iterable **by value** and hand back something that cannot be + /// re-created, so it is made once here and lives until the loop ends. + /// + /// Its table entry is the iterable's *start* position, which is what + /// `ErrorFor` is reported against (`eval/stmt.rs:703`) — a different + /// position from the one [`Op::IterNext`] uses. + IterInit, + + /// Advance the current iterator: push the next item and fall through, or + /// drop the iterator and jump to `exit`. + /// + /// The only instruction whose two edges leave different amounts on the + /// operand stack, which is why the verifier gives it explicit successors. + /// + /// Its table entry is the iterable's position — `position`, not + /// `start_position` — because that is what a fallible iterator's error is + /// filled in with (`eval/stmt.rs:749`). + /// `indexed` is `for (x, i) in seq`: the count is pushed under the item, + /// so the two `StoreShared`s that follow pop them in declaration order. + IterNext { exit: u32, indexed: bool }, + + /// Discard the current iterator. + /// + /// Emitted where a `break` leaves a loop, since the jump skips the + /// [`Op::IterNext`] that would have dropped it on exhaustion. Leaving a + /// frame drops whatever it left behind without this. + IterDrop, + + /// Pop a value and write it into local slot `.0`, through a shared cell + /// rather than over it. + /// + /// Distinct from [`Op::StoreLocal`] only in intent: the `for` loop + /// variable is written once per iteration and a closure in the body may + /// have shared it, in which case rhai writes into the cell and every + /// closure made in the loop sees the last value (`eval/stmt.rs:752`). + StoreShared(u16), + + /// Pop a value and raise it as a `throw`. + /// + /// Always fails, with `ErrorRuntime` carrying the value — rhai wraps + /// nothing and converts nothing, so any type can be thrown. Its table + /// entry is the `throw` keyword's own position, not the expression's + /// (`eval/stmt.rs:877`). + Throw, + + /// End the chunk, yielding the top of the operand stack, or unit if empty. + Return, +} diff --git a/src/grain/bytecode/positions.rs b/src/grain/bytecode/positions.rs new file mode 100644 index 000000000..057a552d4 --- /dev/null +++ b/src/grain/bytecode/positions.rs @@ -0,0 +1,241 @@ +use crate::grain::pos::Site; +use crate::Position; + +/// Where each instruction came from, or nothing. +/// +/// Instructions carry no position of their own. Diagnostics are the one part of +/// an artifact that is never read unless something has already failed, so they +/// are the part worth being able to leave behind — and an instruction that is +/// pure payload is one that can be executed straight out of a borrowed byte +/// slice, which is the larger reason. +/// +/// Moving them out is close to free on bytes: they were only ever stored on the +/// instructions that had one. What it buys is that they can now be removed. +/// `tests/format.rs` measures how much that removes. +/// +/// In memory this is dense, because the lookup is not always cold. A loop +/// back-edge passes a position to `track_operation` on every iteration, and the +/// built-in operator path builds a `NativeCallContext` around one — both on the +/// hot path, both needing it only if something goes wrong. Indexing an array is +/// what makes that free. The compact delta form in [`rhaigrain_pos`] is the +/// wire form, expanded once at load. +/// +/// [`Positions::Stripped`] is not a degraded mode to apologise for: it is what +/// a device runs. Errors come back carrying an instruction address instead of a +/// position, and the host that kept the table resolves it. +#[derive(Debug, Clone, Default)] +pub enum Positions { + /// The table was never written, or was stripped before shipping. + #[default] + Stripped, + /// One entry per instruction, so a lookup is an index. + Dense(Box<[Position]>), +} + +impl Positions { + /// Build from one position per instruction, dropping the table entirely if + /// none of them say anything. + pub(crate) fn dense(positions: Vec) -> Self { + if positions.iter().all(|pos| pos.is_none()) { + return Self::Stripped; + } + Self::Dense(positions.into_boxed_slice()) + } + + /// The position recorded for `pc`, or `Position::NONE`. + /// + /// Out of range reads as no position rather than panicking: this runs while + /// an error is being reported, and losing the position must not replace the + /// error being reported. + #[must_use] + pub fn get(&self, pc: usize) -> Position { + match self { + Self::Stripped => Position::NONE, + Self::Dense(positions) => positions.get(pc).copied().unwrap_or(Position::NONE), + } + } + + /// Whether the position table is stripped + #[must_use] + pub fn is_stripped(&self) -> bool { + matches!(self, Self::Stripped) + } + + /// Encode as the compact table [`rhaigrain_pos::resolve`] reads. + /// + /// Instructions with no position are skipped, which is most of them. + #[must_use] + pub fn to_table(&self) -> Vec { + let Self::Dense(positions) = self else { + return crate::grain::pos::encode(core::iter::empty()); + }; + + crate::grain::pos::encode(positions.iter().enumerate().filter_map(|(pc, pos)| { + let line = pos.line()?; + Some(( + pc as u32, + Site { + line: line as u32, + column: pos.position().unwrap_or(0) as u32, + }, + )) + })) + } + + /// Expand a compact table back to one entry per instruction. + /// + /// `instructions` bounds the result, so a table naming an address past the + /// end of the chunk is refused rather than silently widening the array. + /// + /// # Errors + /// + /// Whatever [`rhaigrain_pos::check`] found, or [`TableError::PastTheEnd`] + /// for an address that does not name an instruction. + pub fn from_table(table: &[u8], instructions: usize) -> Result { + crate::grain::pos::check(table).map_err(TableError::Malformed)?; + + let count = crate::grain::pos::count(table).map_err(TableError::Malformed)?; + if count == 0 { + return Ok(Self::Stripped); + } + + // `check` proved the table sound, so a miss here is an address that + // does not name an instruction in *this* chunk, which the count + // comparison below catches. + let positions: Vec = (0..instructions) + .map(|pc| { + crate::grain::pos::resolve(table, pc as u32) + .map_or(Position::NONE, site_to_position) + }) + .collect(); + + let found = positions.iter().filter(|pos| !pos.is_none()).count(); + if found != count as usize { + return Err(TableError::PastTheEnd { + entries: count as usize, + matched: found, + instructions, + }); + } + + Ok(Self::dense(positions)) + } +} + +/// Why a position table could not be attached to a chunk. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TableError { + /// The table is not well formed. + Malformed(crate::grain::pos::Error), + /// The table names addresses this chunk does not have, so it belongs to a + /// different program. + PastTheEnd { + entries: usize, + matched: usize, + instructions: usize, + }, +} + +impl core::fmt::Display for TableError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Malformed(err) => write!(f, "position table is malformed: {err:?}"), + Self::PastTheEnd { + entries, + matched, + instructions, + } => write!( + f, + "position table has {entries} entries but only {matched} name one of this \ + chunk's {instructions} instructions, so it is a different program's" + ), + } + } +} + +/// A [`Site`] is plain numbers, on purpose — turning it into rhai's own type is +/// this side's job. +fn site_to_position(site: Site) -> Position { + let (Ok(line), Ok(column)) = (u16::try_from(site.line), u16::try_from(site.column)) else { + return Position::NONE; + }; + if line == 0 { + return Position::NONE; + } + Position::new(line, column) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> Positions { + Positions::dense(vec![ + Position::NONE, + Position::new(1, 5), + Position::NONE, + Position::new(3, 0), + ]) + } + + #[test] + fn a_table_survives_the_round_trip() { + let table = sample().to_table(); + let back = Positions::from_table(&table, 4).expect("the table is this chunk's"); + + for pc in 0..4 { + assert_eq!(back.get(pc), sample().get(pc), "at {pc}"); + } + } + + /// Column 0 is the start of a line, not the absence of a position. + #[test] + fn the_start_of_a_line_survives() { + let table = sample().to_table(); + let back = Positions::from_table(&table, 4).unwrap(); + assert_eq!(back.get(3), Position::new(3, 0)); + } + + #[test] + fn stripping_is_what_a_missing_table_reads_as() { + let stripped = Positions::Stripped; + assert!(stripped.is_stripped()); + assert_eq!(stripped.get(0), Position::NONE); + + let empty = Positions::from_table(&stripped.to_table(), 4).unwrap(); + assert!(empty.is_stripped()); + } + + /// A chunk with nothing to say should not carry an array of nothing. + #[test] + fn positions_that_are_all_absent_collapse() { + assert!(Positions::dense(vec![Position::NONE; 8]).is_stripped()); + } + + /// Attaching the wrong program's table would silently misreport every + /// error, which is worse than reporting none. + #[test] + fn a_table_from_another_program_is_refused() { + let table = sample().to_table(); + assert!(matches!( + Positions::from_table(&table, 2), + Err(TableError::PastTheEnd { .. }), + )); + } + + #[test] + fn a_malformed_table_is_refused_rather_than_half_applied() { + let table = sample().to_table(); + assert!(matches!( + Positions::from_table(&table[..table.len() - 1], 4), + Err(TableError::Malformed(..)), + )); + } + + /// Reading past the chunk is how an error report finds out there is no + /// position, and it happens while another error is already in flight. + #[test] + fn an_address_past_the_end_reads_as_no_position() { + assert_eq!(sample().get(9999), Position::NONE); + } +} diff --git a/src/grain/bytecode/strings.rs b/src/grain/bytecode/strings.rs new file mode 100644 index 000000000..2ca3cd1b1 --- /dev/null +++ b/src/grain/bytecode/strings.rs @@ -0,0 +1,215 @@ +use std::borrow::Cow; + +/// Every name a program mentions, as one blob and a list of spans. +/// +/// Names are the largest thing a load used to allocate: an `ImmutableString` +/// is a reference-counted box, so a hundred of them is a hundred allocations +/// and several kilobytes. Almost none of them need to be one — a call name, an +/// operator, a getter and a property key are all handed to rhai as `&str`. +/// +/// So the table is a byte blob borrowed straight out of the artifact, and a +/// name is a slice of it. Two allocations for the whole table, and neither +/// scales with how many names there are: the spans, and nothing else. +/// +/// The exception is a name that becomes a scope entry. `Scope` stores its own +/// names, so a `let` or a parameter needs a real string — see +/// [`Program::ident`](crate::Program::ident), which is a separate and much +/// shorter pool. +#[derive(Debug, Clone, Default)] +pub struct Strings<'a> { + blob: Cow<'a, [u8]>, + /// Start of each name; the end is the next start, so this is one longer + /// than the number of names. + starts: Vec, +} + +impl<'a> Strings<'a> { + /// Build from names in index order. + #[must_use] + pub fn new>(names: impl IntoIterator) -> Strings<'static> { + let mut blob = Vec::new(); + let mut starts = vec![0u32]; + + for name in names { + blob.extend_from_slice(name.as_ref().as_bytes()); + starts.push(blob.len() as u32); + } + + Strings { + blob: Cow::Owned(blob), + starts, + } + } + + /// Wrap a blob and its spans, as read from an artifact. + /// + /// # Errors + /// + /// [`BadTable`] if the spans do not ascend within the blob, or if any name + /// is not UTF-8 — both of which would otherwise turn into a slice panic + /// while the VM was already running. + pub fn borrowed(blob: &'a [u8], starts: Vec) -> Result { + if starts.is_empty() { + return Err(BadTable::NoTerminator); + } + if starts[0] != 0 { + return Err(BadTable::NoTerminator); + } + + for pair in starts.windows(2) { + let (from, to) = (pair[0] as usize, pair[1] as usize); + if to < from || to > blob.len() { + return Err(BadTable::SpanOutOfRange { from, to }); + } + core::str::from_utf8(&blob[from..to]).map_err(|_| BadTable::NotUtf8 { at: from })?; + } + if *starts.last().expect("checked") as usize != blob.len() { + return Err(BadTable::TrailingBytes); + } + + Ok(Self { + blob: Cow::Borrowed(blob), + starts, + }) + } + + #[must_use] + pub fn len(&self) -> usize { + self.starts.len().saturating_sub(1) + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The name at `index`, or `None`. + /// + /// Never allocates: the result points into the artifact. + #[must_use] + pub fn get(&self, index: u32) -> Option<&str> { + let from = *self.starts.get(index as usize)? as usize; + let to = *self.starts.get(index as usize + 1)? as usize; + // Checked once on construction, so this cannot fail. + core::str::from_utf8(self.blob.get(from..to)?).ok() + } + + #[must_use] + pub fn blob(&self) -> &[u8] { + &self.blob + } + + #[must_use] + pub fn starts(&self) -> &[u32] { + &self.starts + } + + pub fn iter(&self) -> impl Iterator + '_ { + (0..self.len() as u32).filter_map(|index| self.get(index)) + } + + #[must_use] + pub fn into_owned(self) -> Strings<'static> { + Strings { + blob: Cow::Owned(self.blob.into_owned()), + starts: self.starts, + } + } +} + +/// Why a string table could not be used. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BadTable { + /// The spans are empty, or do not start at zero. + NoTerminator, + /// A span runs backwards or past the end of the blob. + SpanOutOfRange { from: usize, to: usize }, + /// A name is not UTF-8. + NotUtf8 { at: usize }, + /// The blob is longer than the last span accounts for. + TrailingBytes, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_come_back_in_order() { + let table = Strings::new(["alpha", "b", "", "gamma"]); + assert_eq!(table.len(), 4); + assert_eq!(table.get(0), Some("alpha")); + assert_eq!(table.get(1), Some("b")); + assert_eq!(table.get(2), Some(""), "an empty name is still a name"); + assert_eq!(table.get(3), Some("gamma")); + assert_eq!(table.get(4), None); + } + + #[test] + fn a_table_survives_being_taken_apart_and_borrowed_back() { + let owned = Strings::new(["one", "two", "three"]); + let blob = owned.blob().to_vec(); + + let borrowed = Strings::borrowed(&blob, owned.starts().to_vec()).expect("sound"); + assert_eq!(borrowed.iter().collect::>(), ["one", "two", "three"]); + } + + #[test] + fn an_empty_table_is_sound() { + let table = Strings::new(Vec::<&str>::new()); + assert!(table.is_empty()); + assert_eq!(table.get(0), None); + + let borrowed = Strings::borrowed(&[], vec![0]).expect("sound"); + assert!(borrowed.is_empty()); + } + + /// The spans come from an artifact, so they are untrusted and must be + /// rejected rather than turned into a slice panic mid-run. + #[test] + fn spans_that_do_not_fit_the_blob_are_refused() { + assert_eq!( + Strings::borrowed(b"abc", vec![]).err(), + Some(BadTable::NoTerminator), + ); + assert_eq!( + Strings::borrowed(b"abc", vec![1, 3]).err(), + Some(BadTable::NoTerminator), + ); + assert!(matches!( + Strings::borrowed(b"abc", vec![0, 99]), + Err(BadTable::SpanOutOfRange { .. }), + )); + assert!(matches!( + Strings::borrowed(b"abc", vec![0, 2, 1]), + Err(BadTable::SpanOutOfRange { .. }), + )); + assert_eq!( + Strings::borrowed(b"abcd", vec![0, 3]).err(), + Some(BadTable::TrailingBytes), + ); + } + + #[test] + fn a_name_that_is_not_utf8_is_refused() { + assert!(matches!( + Strings::borrowed(&[0xff, 0xfe], vec![0, 2]), + Err(BadTable::NotUtf8 { .. }), + )); + } + + /// The property the whole type exists for. + #[test] + fn borrowing_points_into_the_caller_s_buffer() { + let blob = b"alphabeta".to_vec(); + let table = Strings::borrowed(&blob, vec![0, 5, 9]).expect("sound"); + + let name = table.get(0).expect("present"); + assert_eq!(name, "alpha"); + assert!( + name.as_ptr() >= blob.as_ptr() + && (name.as_ptr() as usize) < blob.as_ptr() as usize + blob.len(), + "a borrowed name must point into the blob, not into a copy", + ); + } +} diff --git a/src/grain/bytecode/switch.rs b/src/grain/bytecode/switch.rs new file mode 100644 index 000000000..c3ca5a00c --- /dev/null +++ b/src/grain/bytecode/switch.rs @@ -0,0 +1,284 @@ +use crate::{ast::RangeCase, Dynamic, INT}; + +/// A `switch`, as one dispatch table. +/// +/// Matching a case is *hash* equality, not `==`. That distinction is rhai's +/// and it is visible: `switch 1 { 1.0 => .. }` does not match, because an +/// integer and a float hash differently, while `1 == 1.0` is true. +/// +/// ## Why the hashes travel, and what that costs +/// +/// Rhai's parser keeps only the hash of each case — the value itself is not in +/// the AST (`ast/stmt.rs:336`), so there is nothing to re-hash later. The +/// hashes have to be written out as they are. +/// +/// And by default they do not survive the trip: `get_hasher` falls back to +/// `ahash::AHasher::default()`, and rhai's default features include +/// `ahash/runtime-rng`, so the seed is drawn per process. Rhai gets away with +/// baking hashes into its AST only because it parses and evaluates in one. +/// +/// So an artifact containing a `switch` requires +/// [`rhai::config::hashing::set_hashing_seed`] to have been called with the +/// same seed on both sides. That is not something the format can enforce, but +/// it is something it can *check*: [`probe`] hashes a fixed value, the +/// artifact carries the result, and a loader that computes a different one +/// refuses rather than dispatching every case to the default. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Switch { + /// One entry per distinct case value, in source order. The target is the + /// head of that value's chain of guarded arms. + pub cases: Vec, + /// Checked only when no case matched at all, as rhai does — a case that + /// matched but whose guards all failed goes to the default rather than on + /// to the ranges (`eval/stmt.rs:544`). + /// + /// Disjoint and in ascending order, which rhai's are not: the compiler + /// splits overlapping arms apart so that the first entry containing a + /// value is the only one that can match it. See + /// [`crate::compile::cases`]. + pub ranges: Vec, + /// Where to go when nothing matched. Always present: an absent `_` arm + /// compiles to a jump past the statement. + pub default: u32, +} + +/// One `value => ...` arm, keyed by rhai's hash of the value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SwitchCase { + pub hash: u64, + pub target: u32, +} + +/// One `a..b => ...` arm. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SwitchRange { + pub from: INT, + pub to: INT, + pub inclusive: bool, + pub target: u32, +} + +impl SwitchRange { + /// Whether a subject falls in this range. + /// + /// Delegates to rhai's own [`RangeCase`] rather than comparing integers, + /// because a range arm matches more than integers: `switch 5.5 { 0..10 => + /// .. }` matches, and under the `decimal` feature so does a `Decimal` + /// (`ast/stmt.rs:254`). Rebuilding the case is two moves and no + /// allocation, and it means there is one definition of what a range arm + /// covers. + #[must_use] + pub fn contains(&self, value: &Dynamic) -> bool { + let case: RangeCase = if self.inclusive { + (self.from..=self.to).into() + } else { + (self.from..self.to).into() + }; + case.contains(value) + } +} + +impl Switch { + /// Where a subject sends control. + /// + /// The order is rhai's (`eval/stmt.rs:517-564`) and each step of it is + /// load-bearing: an unhashable subject reaches neither the cases nor the + /// ranges, and a subject whose hash *did* find a case never reaches the + /// ranges even when that case's guards all decline it — the compiler + /// points such a chain at the default. + #[must_use] + pub fn dispatch(&self, subject: &Dynamic) -> u32 { + // Hashing an unhashable value panics, so this is a guard and not an + // optimization. + if !subject.is_hashable() { + return self.default; + } + + let hash = hash_of(subject); + if let Some(case) = self.cases.iter().find(|case| case.hash == hash) { + return case.target; + } + + // Disjoint, so the first containing entry is the only one. + if let Some(range) = self.ranges.iter().find(|r| r.contains(subject)) { + return range.target; + } + + self.default + } +} + +/// A fixed value hashed with the engine's hasher, so two processes can find +/// out whether their case hashes mean the same thing. +/// +/// Not a checksum of the seed — the seed is not readable as a number the +/// format could compare. This is the observable consequence of it. +#[must_use] +pub fn probe() -> u64 { + hash_of(&Dynamic::from("rhaigrain switch probe")) +} + +fn hash_of(value: &Dynamic) -> u64 { + use core::hash::{Hash, Hasher}; + + let mut hasher = crate::func::get_hasher(); + value.hash(&mut hasher); + hasher.finish() +} + +/// The hash rhai's `switch` would key `value` under. +/// +/// Test-only. The compiler never hashes anything: rhai's parser has already +/// grouped the arms by hash, and the hashes are all it kept. +#[cfg(test)] +fn case_hash(value: &Dynamic) -> Option { + value.is_hashable().then(|| hash_of(value)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A host type, which has no hash. + #[derive(Debug, Clone)] + struct Opaque; + + fn table(cases: &[(&Dynamic, u32)], ranges: Vec, default: u32) -> Switch { + Switch { + cases: cases + .iter() + .filter_map(|(value, target)| { + Some(SwitchCase { + hash: case_hash(value)?, + target: *target, + }) + }) + .collect(), + ranges, + default, + } + } + + #[test] + fn a_matching_case_wins() { + let (one, two) = (Dynamic::from(1_i64), Dynamic::from(2_i64)); + let table = table(&[(&one, 10), (&two, 20)], Vec::new(), 99); + + assert_eq!(table.dispatch(&one), 10); + assert_eq!(table.dispatch(&two), 20); + assert_eq!(table.dispatch(&Dynamic::from(3_i64)), 99); + } + + /// The distinction that makes this hashing rather than `==`: rhai does not + /// match an integer against a float case, even though `1 == 1.0`. + #[test] + fn a_float_does_not_match_an_integer_case() { + let one = Dynamic::from(1_i64); + let table = table(&[(&one, 10)], Vec::new(), 99); + assert_eq!(table.dispatch(&Dynamic::from(1.0_f64)), 99); + } + + #[test] + fn strings_and_characters_match_by_value() { + let (text, ch, flag) = ( + Dynamic::from("hello"), + Dynamic::from('x'), + Dynamic::from(true), + ); + let table = table(&[(&text, 10), (&ch, 20), (&flag, 30)], Vec::new(), 99); + + assert_eq!(table.dispatch(&Dynamic::from("hello")), 10); + assert_eq!(table.dispatch(&Dynamic::from("other")), 99); + assert_eq!(table.dispatch(&ch), 20); + assert_eq!(table.dispatch(&flag), 30); + } + + /// A range arm covers the reals between its bounds, not just the integers + /// in them — which is why the check delegates to rhai's own `RangeCase` + /// rather than comparing integers. + #[test] + #[cfg(not(feature = "no_float"))] + fn a_range_catches_a_float_between_its_bounds() { + let table = table( + &[], + vec![SwitchRange { + from: 0, + to: 10, + inclusive: false, + target: 20, + }], + 99, + ); + + // The script float type, not `f64`: under `f32_float` a `Dynamic` + // holding an `f64` is a foreign type and never matches a range arm. + assert_eq!(table.dispatch(&Dynamic::from(5.5 as rhai::FLOAT)), 20); + assert_eq!( + table.dispatch(&Dynamic::from(10.0 as rhai::FLOAT)), + 99, + "exclusive end" + ); + assert_eq!(table.dispatch(&Dynamic::from(-0.5 as rhai::FLOAT)), 99); + } + + /// Ranges are consulted only after the cases miss. + #[test] + fn a_range_catches_what_no_case_did() { + let one = Dynamic::from(1_i64); + let table = table( + &[(&one, 10)], + vec![ + SwitchRange { + from: 5, + to: 8, + inclusive: false, + target: 20, + }, + SwitchRange { + from: 8, + to: 10, + inclusive: true, + target: 30, + }, + ], + 99, + ); + + assert_eq!(table.dispatch(&one), 10, "a case still wins"); + assert_eq!(table.dispatch(&Dynamic::from(5_i64)), 20); + assert_eq!(table.dispatch(&Dynamic::from(7_i64)), 20); + assert_eq!(table.dispatch(&Dynamic::from(8_i64)), 30, "exclusive end"); + assert_eq!(table.dispatch(&Dynamic::from(10_i64)), 30, "inclusive end"); + assert_eq!(table.dispatch(&Dynamic::from(11_i64)), 99); + } + + /// Hashing one would panic, so it must never reach the hasher — and it + /// must still be able to reach the default. + #[test] + fn an_unhashable_subject_falls_through_rather_than_panicking() { + let one = Dynamic::from(1_i64); + let table = table(&[(&one, 10)], Vec::new(), 99); + + // A bare function pointer *is* hashable; only one carrying an + // environment is not. A host type is the reliable case. + let unhashable = Dynamic::from(Opaque); + assert!( + !unhashable.is_hashable(), + "this test needs an unhashable value", + ); + assert_eq!(table.dispatch(&unhashable), 99); + } + + #[test] + fn an_unhashable_case_has_no_hash_to_key_on() { + assert_eq!(case_hash(&Dynamic::from(Opaque)), None); + assert!(case_hash(&Dynamic::from(1_i64)).is_some()); + } + + /// The probe is only worth carrying if it actually depends on the seed. + #[test] + fn the_probe_is_stable_within_a_process() { + assert_eq!(probe(), probe()); + assert_ne!(probe(), 0, "a probe of zero could not be told from absent"); + } +} diff --git a/src/grain/bytecode/verify.rs b/src/grain/bytecode/verify.rs new file mode 100644 index 000000000..a306448a9 --- /dev/null +++ b/src/grain/bytecode/verify.rs @@ -0,0 +1,906 @@ +use crate::grain::bytecode::code::{self, tag}; +use crate::grain::bytecode::{Chain, Chunk, Op, Receiver, Root, Step, Switch, Tail}; + +/// What the pools hold, so an instruction's indices can be checked against +/// something. +/// +/// Chains and switches come through whole rather than as a count, because both +/// hold things that have to be checked rather than counted: how much operand +/// stack a chain consumes, and where a switch can send control. +#[derive(Debug, Clone, Copy)] +pub struct Pools<'a> { + pub consts: usize, + pub names: usize, + pub tokens: usize, + pub assign_ops: usize, + pub residuals: usize, + pub chains: &'a [Chain], + pub switches: &'a [Switch], +} + +/// Why a chunk was rejected. +/// +/// Every variant names something a correct compiler cannot produce, so a +/// failure here is a bug in the compiler or a corrupted artifact — never +/// anything a script can express. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VerifyError { + /// A tag with no instruction behind it, or one whose operands run past the + /// end of the chunk. + Undecodable { at: usize }, + /// The last instruction stops short of the end, so the trailing bytes are + /// not instructions. + TrailingBytes { at: usize, len: usize }, + /// A chunk names a span the code does not have. + ChunkOutOfRange { entry: u32, end: u32, len: usize }, + /// A jump leaves the chunk it is in — including into another chunk, which + /// would run that function's instructions against this frame's locals. + JumpOutOfRange { at: usize, target: u32 }, + /// A jump lands inside an instruction rather than on one. Decoding from + /// there would read an operand's bytes as a tag. + JumpIntoAnInstruction { at: usize, target: u32 }, + /// Two paths reach the same instruction with different stack depths, so + /// the depth at that point is not a static property. + DepthConflict { + at: usize, + expected: usize, + found: usize, + }, + /// An instruction pops more than is on the stack. + Underflow { at: usize, need: usize, have: usize }, + /// Execution can run past the last instruction. + FallsOffTheEnd, + /// The chunk claims less stack than it uses. + StackExceedsDeclared { needed: usize, declared: u16 }, + /// An iterator is dropped where none was made. The compiler pairs these + /// up lexically; an artifact off a wire has to be asked. + IteratorUnderflow { at: usize }, + /// A handler is disarmed where none was armed. A stale handler is worse + /// than a missing one: the next unrelated error would be caught into an + /// already-exited `catch` block. + HandlerUnderflow { at: usize }, + /// An index into a pool with nothing behind it. + BadIndex { + at: usize, + what: &'static str, + index: u32, + }, +} + +/// Check that a chunk is internally consistent before running it. +/// +/// Two passes. The first decodes straight through, recording where each +/// instruction starts; that is what makes it safe to say a jump target is or is +/// not an instruction, which a reachability walk alone cannot — a jump into the +/// middle of an operand would decode the operand's bytes as a tag and look +/// perfectly reasonable. +/// +/// The second is an abstract interpretation over stack depth: walk every +/// reachable instruction, and require that all paths into one agree on how deep +/// the operand stack is. That single property catches the whole class of +/// compiler bugs where one branch of a conditional leaves a value and the other +/// does not — which is otherwise invisible until a program takes the unlucky +/// path. +/// +/// Together they are what makes an artifact safe to execute in place: a chunk +/// that passes cannot underflow the operand stack, jump outside itself, or +/// decode an operand as an instruction. What it does *not* prove is +/// termination — a jump target inside the chunk is well-formed whether or not +/// it closes a loop — which is why [`Op::Tick`] sits on every back edge and why +/// a host running untrusted bytecode still needs `max_operations`. +/// +/// Returns the measured stack high water, which is what the chunk should +/// declare. +pub fn verify(code: &[u8], chunks: &[Chunk], pools: Pools) -> Result, VerifyError> { + // Pass one: where do instructions start? + // + // Over the whole buffer at once, because every chunk shares it and an + // instruction boundary is a property of the bytes, not of who runs them. + let mut starts = vec![false; code.len() + 1]; + let mut at = 0usize; + while at < code.len() { + starts[at] = true; + let width = code::width(code, at).ok_or(VerifyError::Undecodable { at })?; + check_indices(at, code, pools)?; + at += width; + } + if at != code.len() { + return Err(VerifyError::TrailingBytes { + at, + len: code.len(), + }); + } + + chunks + .iter() + .map(|chunk| verify_chunk(code, chunk, &starts, pools)) + .collect() +} + +/// What every path into an instruction has to agree on. +/// +/// The operand stack is the obvious one. The iterator stack is here for the +/// same reason: a `for` loop's iterator lives on a stack of the VM's own, and +/// a chunk that leaves one behind — or drops one it never made — is a chunk +/// whose loops are not the shape the compiler thought. "The compiler balances +/// them" is exactly the sort of claim a verifier for untrusted bytecode exists +/// to check rather than take on trust. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct State { + operands: usize, + iters: usize, + handlers: usize, +} + +/// Walk one chunk's reachable instructions, checking that every path into an +/// instruction agrees on the stack depth. +fn verify_chunk( + code: &[u8], + chunk: &Chunk, + starts: &[bool], + pools: Pools, +) -> Result { + let (entry, end) = (chunk.entry() as usize, chunk.end() as usize); + if end > code.len() || entry > end { + return Err(VerifyError::ChunkOutOfRange { + entry: chunk.entry(), + end: chunk.end(), + len: code.len(), + }); + } + + let mut depth_at: Vec> = vec![None; code.len()]; + let mut worklist = vec![(entry, State::default())]; + let mut high_water = 0usize; + + while let Some((at, state)) = worklist.pop() { + if at >= end { + return Err(VerifyError::FallsOffTheEnd); + } + + // Merge point: either this is the first visit, or every earlier path + // has to have arrived in the same state. + match depth_at[at] { + Some(seen) if seen == state => continue, + Some(seen) => { + return Err(VerifyError::DepthConflict { + at, + expected: seen.operands, + found: state.operands, + }) + } + None => depth_at[at] = Some(state), + } + + let depth = state.operands; + high_water = high_water.max(depth); + + let op = code::decode(code, at).ok_or(VerifyError::Undecodable { at })?; + let (pops, pushes) = effect(&op, pools); + + if depth < pops { + return Err(VerifyError::Underflow { + at, + need: pops, + have: depth, + }); + } + let next_state = State { + operands: depth - pops + pushes, + iters: match op { + Op::IterInit => state.iters + 1, + Op::IterDrop => state + .iters + .checked_sub(1) + .ok_or(VerifyError::IteratorUnderflow { at })?, + _ => state.iters, + }, + handlers: match op { + Op::PushHandler { .. } => state.handlers + 1, + Op::PopHandler => state + .handlers + .checked_sub(1) + .ok_or(VerifyError::HandlerUnderflow { at })?, + _ => state.handlers, + }, + }; + let next_depth = next_state.operands; + high_water = high_water.max(next_depth); + + let width = code::width(code, at).expect("decoded, so it has a width"); + let next = at + width; + + let mut go = |target: u32, state: State| -> Result<(), VerifyError> { + let target = target as usize; + // Within this chunk: a jump into another function's body would run + // its instructions against this frame's locals. + if target < entry || target >= end { + return Err(VerifyError::JumpOutOfRange { + at, + target: target as u32, + }); + } + if !starts[target] { + return Err(VerifyError::JumpIntoAnInstruction { + at, + target: target as u32, + }); + } + worklist.push((target, state)); + Ok(()) + }; + + match op { + // Terminal: nothing follows. A `throw` always fails, so control + // leaves the chunk here as surely as it does at a `Return`. + // + // Neither has to balance the iterator stack: leaving a frame + // truncates it to what the frame started with. + Op::Return | Op::Throw => {} + + Op::Jump(target) => go(target, next_state)?, + + // The catch block is entered on the exception path, so it starts + // where the `try` did: same operand depth, same iterators, and + // inside the handler it will disarm itself. + Op::PushHandler { target, .. } => { + go( + target, + State { + operands: depth, + iters: state.iters, + handlers: next_state.handlers, + }, + )?; + worklist.push((next, next_state)); + } + + Op::JumpIfFalse { target } | Op::JumpIfTrue { target } => { + go(target, next_state)?; + worklist.push((next, next_state)); + } + + // The one instruction whose edges differ in more than where they + // go: falling through carries the item it pushed and still holds + // the iterator, while the exit edge has neither. + Op::IterNext { exit, indexed } => { + go( + exit, + State { + operands: depth, + iters: state + .iters + .checked_sub(1) + .ok_or(VerifyError::IteratorUnderflow { at })?, + handlers: state.handlers, + }, + )?; + worklist.push(( + next, + State { + // The item, and the count under it when there is one. + operands: depth + 1 + usize::from(indexed), + iters: state.iters, + handlers: state.handlers, + }, + )); + } + + // Terminal like `Jump`, with one successor per arm. A table with + // no entry behind it is caught by `check_indices`, which has + // already run over every instruction. + Op::Switch(index) => { + if let Some(table) = pools.switches.get(index as usize) { + for target in table + .cases + .iter() + .map(|case| case.target) + .chain(table.ranges.iter().map(|range| range.target)) + .chain(core::iter::once(table.default)) + { + go(target, next_state)?; + } + } + } + + _ => { + if next >= end { + return Err(VerifyError::FallsOffTheEnd); + } + worklist.push((next, next_state)); + } + } + } + + let Ok(high_water) = u16::try_from(high_water) else { + return Err(VerifyError::StackExceedsDeclared { + needed: high_water, + declared: chunk.max_stack(), + }); + }; + if high_water > chunk.max_stack() { + return Err(VerifyError::StackExceedsDeclared { + needed: high_water as usize, + declared: chunk.max_stack(), + }); + } + + Ok(high_water) +} + +/// How many operands an instruction consumes and produces. +fn effect(op: &Op, pools: Pools) -> (usize, usize) { + match op { + // A chain eats the indices and arguments its steps named, plus a root + // that is not a slot, plus the value being assigned, and leaves one + // behind. An index with no chain behind it reads as consuming nothing; + // `check_indices` is what rejects it. + Op::Chain(index) => match pools.chains.get(*index as usize) { + Some(chain) => (chain.consumes(), 1), + None => (0, 1), + }, + + Op::Const(..) + | Op::Unit + | Op::Bool(..) + | Op::LoadLocal(..) + | Op::LoadNamed(..) + | Op::LoadShared(..) + | Op::LoadSharedNamed(..) + | Op::MakeClosure(..) + | Op::EvalAst { .. } => (0, 1), + + // Sharing is a change to the scope, not to the operand stack. + Op::Share(..) | Op::ShareNamed(..) => (0, 0), + + Op::StoreLocal(..) | Op::DeclareLocal { .. } | Op::Pop => (1, 0), + + // Pops the value, leaves nothing: the statement's unit value is a + // separate `Op::Unit`. + Op::AssignLocal { .. } | Op::AssignNamed { .. } => (1, 0), + + Op::JumpIfFalse { .. } | Op::JumpIfTrue { .. } | Op::Switch(..) => (1, 0), + + Op::Jump(..) + | Op::UnwindTo(..) + | Op::Tick + | Op::Checkpoint + | Op::PushHandler { .. } + | Op::PopHandler => (0, 0), + + // Arguments in, result out. + Op::Call { argc, .. } => (*argc as usize, 1), + + // A named receiver's value is argument zero like any other; a local's + // is not on the stack at all. An `argc` of zero names no receiver and + // is rejected when it runs. + Op::CallRef { argc, receiver, .. } => match receiver { + Receiver::Local(..) => ((*argc as usize).saturating_sub(1), 1), + Receiver::Named(..) => (*argc as usize, 1), + }, + + // Reorders what is already there, so the depth is unchanged — but it + // has to reach every one of them, and saying so is what stops a + // hand-made artifact reaching under the frame. + Op::Rotate(under) => (*under as usize + 1, *under as usize + 1), + + Op::MakeArray(len) => (*len as usize, 1), + + // A key and a value per entry, and the template underneath them. + Op::MakeMap(len) => (2 * *len as usize + 1, 1), + + // Measures the element it is standing on without taking it: the + // literal is still being built and every element it has so far is + // still on the stack. + Op::CheckSize { .. } => (1, 1), + + // The buffer is an ordinary operand: started, appended to, then + // replaced by the string it built. + // A name in, a pointer out. + Op::MakeFnPtr | Op::IsShared => (1, 1), + // The arguments and the pointer itself, leaving one of each. + Op::Curry(argc) => (*argc as usize + 1, 1), + Op::CallFnPtr { argc, .. } => (*argc as usize + 1, 1), + + Op::InterpolateStart => (0, 1), + Op::InterpolateAppend => (1, 0), + Op::InterpolateEnd => (1, 1), + + // Pops the thrown value; nothing follows, so what it leaves is moot. + Op::Throw | Op::StoreShared(..) => (1, 0), + + // The iterable goes onto the iterator stack, not back onto this one. + Op::IterInit => (1, 0), + // Its two edges disagree, so the successor match does the work. + Op::IterNext { .. } | Op::IterDrop => (0, 0), + + // Consumes whatever is left, so depth afterwards is not meaningful. + Op::Return => (0, 0), + } +} + +/// Check that every pool reference resolves. +/// +/// The VM treats these as assertions, and an artifact is the one place they can +/// be wrong without a compiler bug. Reads the operands off the bytes rather +/// than off a decoded `Op`, so it runs in the same pass that measures widths. +fn check_indices(at: usize, code: &[u8], pools: Pools) -> Result<(), VerifyError> { + let index = |offset: usize| code::u16_at(code, at + offset).map_or(0, u32::from); + let bounded = |index: u32, what: &'static str, len: usize| { + if index as usize >= len { + Err(VerifyError::BadIndex { at, what, index }) + } else { + Ok(()) + } + }; + + match code[at] { + tag::CONST => bounded(index(1), "constant", pools.consts), + tag::DECLARE_LOCAL | tag::DECLARE_CONST => bounded(index(1), "name", pools.names), + tag::CALL | tag::CALL_LOCAL_REF => bounded(index(1), "name", pools.names), + // The function's, then the receiver variable's. The slot a local + // receiver names is not a pool index and is checked against the scope + // when it runs, as every other slot is. + tag::CALL_NAMED_REF => { + bounded(index(1), "name", pools.names)?; + bounded(index(4), "name", pools.names) + } + tag::CALL_OP => { + bounded(index(1), "name", pools.names)?; + bounded(index(4), "operator", pools.tokens) + } + tag::ASSIGN_LOCAL => bounded(index(3), "name", pools.names), + tag::LOAD_NAMED + | tag::LOAD_SHARED_NAMED + | tag::ASSIGN_NAMED + | tag::SHARE_NAMED + | tag::MAKE_CLOSURE => bounded(index(1), "name", pools.names), + tag::ASSIGN_NAMED_OP => { + bounded(index(1), "name", pools.names)?; + bounded(index(3), "op-assignment", pools.assign_ops) + } + tag::ASSIGN_LOCAL_OP => { + bounded(index(3), "name", pools.names)?; + bounded(index(5), "op-assignment", pools.assign_ops) + } + tag::EVAL_AST | tag::EVAL_AST_KEEP => bounded(index(1), "fragment", pools.residuals), + tag::CHAIN => { + bounded(index(1), "chain", pools.chains.len())?; + check_chain_indices(at, &pools.chains[index(1) as usize], pools) + } + tag::SWITCH => bounded(index(1), "switch", pools.switches.len()), + _ => Ok(()), + } +} + +/// Check the pool references *inside* a chain record. +/// +/// A chain is one instruction over an unbounded record, so nearly all of what +/// it names lives in the pool rather than in the code. Bounding only the +/// record's own index would leave most of the instruction unverified. +fn check_chain_indices(at: usize, chain: &Chain, pools: Pools) -> Result<(), VerifyError> { + let bounded = |index: u32, what: &'static str, len: usize| { + if index as usize >= len { + Err(VerifyError::BadIndex { at, what, index }) + } else { + Ok(()) + } + }; + + match chain.root { + Root::Local { name, .. } | Root::Named { name, .. } => { + bounded(name, "name", pools.names)?; + } + Root::Temporary => {} + } + + for step in &chain.steps { + match step { + // Its operands are stack offsets and its positions are its own. + Step::Index { .. } => {} + Step::Property { + name, + getter, + setter, + .. + } => { + bounded(*name, "name", pools.names)?; + bounded(*getter, "name", pools.names)?; + bounded(*setter, "name", pools.names)?; + } + Step::Method { name, .. } => bounded(*name, "name", pools.names)?, + } + } + + match chain.tail { + Tail::Assign { op: Some(op) } => bounded(op, "op-assignment", pools.assign_ops), + Tail::Assign { op: None } | Tail::Read => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::grain::bytecode::assemble; + + fn pools() -> Pools<'static> { + Pools { + consts: 0, + names: 0, + tokens: 0, + assign_ops: 0, + residuals: 0, + chains: &[], + switches: &[], + } + } + + /// Assemble one chunk spanning the whole buffer, and check it. + fn check(ops: Vec) -> Result, VerifyError> { + let (code, _) = assemble(&ops).expect("the test ops must assemble"); + let chunk = Chunk::new(0, code.len() as u32, 8); + verify(&code, &[chunk], pools()) + } + + /// The same, for bytes `assemble` would refuse to produce — which is what + /// a corrupt artifact hands the loader. + fn check_bytes(code: Vec, max_stack: u16) -> Result, VerifyError> { + let chunk = Chunk::new(0, code.len() as u32, max_stack); + verify(&code, &[chunk], pools()) + } + + #[test] + fn accepts_a_well_formed_chunk() { + assert_eq!(check(vec![Op::Unit, Op::Return]), Ok(vec![1])); + } + + /// A chain is one instruction over an unbounded record, so almost all of it + /// is in the pool rather than in the code. Checking only the record's own + /// index would leave the rest of the instruction unverified. + #[test] + fn rejects_a_chain_that_names_something_the_pools_do_not_have() { + let chain = |root, steps, tail| Chain { + root, + steps, + tail, + operands: 0, + }; + let property = |name| Step::Property { + name, + getter: 0, + setter: 0, + pos: rhai::Position::NONE, + }; + + let past_the_end = [ + chain( + Root::Named { + name: 3, + pos: rhai::Position::NONE, + }, + vec![], + Tail::Read, + ), + chain(Root::Local { slot: 0, name: 3 }, vec![], Tail::Read), + chain(Root::Temporary, vec![property(3)], Tail::Read), + chain( + Root::Temporary, + vec![Step::Method { + name: 3, + argc: 0, + operand: 0, + pos: rhai::Position::NONE, + }], + Tail::Read, + ), + ]; + + for chain in past_the_end { + let temporary = chain.roots_on_stack(); + let mut ops = vec![Op::Chain(0), Op::Return]; + if temporary { + ops.insert(0, Op::Unit); + } + let (code, _) = assemble(&ops).expect("must assemble"); + let chunk = Chunk::new(0, code.len() as u32, 8); + let pools = Pools { + names: 1, + chains: std::slice::from_ref(&chain), + ..pools() + }; + assert!( + matches!( + verify(&code, &[chunk], pools), + Err(VerifyError::BadIndex { + what: "name", + index: 3, + .. + }), + ), + "{chain:?} names name 3 of 1 and must be refused", + ); + } + + // And the op-assignment a tail can carry. + let assigning = chain( + Root::Local { slot: 0, name: 0 }, + vec![], + Tail::Assign { op: Some(2) }, + ); + let (code, _) = assemble(&[Op::Unit, Op::Chain(0), Op::Return]).expect("must assemble"); + let chunk = Chunk::new(0, code.len() as u32, 8); + assert!(matches!( + verify( + &code, + &[chunk], + Pools { + names: 1, + chains: std::slice::from_ref(&assigning), + ..pools() + }, + ), + Err(VerifyError::BadIndex { + what: "op-assignment", + index: 2, + .. + }), + )); + } + + /// Chunks share one buffer, so a jump from one into another would run the + /// callee's instructions against the caller's locals. + #[test] + fn rejects_a_jump_from_one_chunk_into_another() { + // Two chunks: `Unit; Return` twice. The first jumps into the second. + let (mut code, _) = assemble(&[Op::Unit, Op::Return]).unwrap(); + let boundary = code.len() as u32; + code.push(tag::JUMP); + code.extend_from_slice(&0u32.to_le_bytes()); // back into chunk one + code.push(tag::RETURN); + + let chunks = [ + Chunk::new(0, boundary, 8), + Chunk::new(boundary, code.len() as u32, 8), + ]; + assert!(matches!( + verify(&code, &chunks, pools()), + Err(VerifyError::JumpOutOfRange { .. }), + )); + } + + /// The reason the high water is returned rather than merely checked: the + /// compiler's estimate is one slot per instruction, and the VM reserves + /// from it. + #[test] + fn the_high_water_is_what_the_chunk_uses_not_what_it_declares() { + assert_eq!( + check(vec![ + Op::Unit, + Op::Unit, + Op::Pop, + Op::Pop, + Op::Unit, + Op::Return + ]), + Ok(vec![2]), + ); + } + + /// The property the verifier exists for: one branch leaves a value, the + /// other does not, and nothing notices until a program takes the wrong + /// path at runtime. + #[test] + fn rejects_branches_that_disagree_on_depth() { + let ops = vec![ + Op::Bool(true), + Op::JumpIfFalse { target: 3 }, + Op::Unit, // the taken path pushes + Op::Return, + ]; + + assert!( + matches!(check(ops.clone()), Err(VerifyError::DepthConflict { .. })), + "a branch imbalance must be rejected, got {:?}", + check(ops), + ); + } + + #[test] + fn rejects_a_jump_off_the_end() { + // Assembled by hand: `assemble` refuses an index it cannot resolve, so + // an out-of-range *address* can only come from a corrupt artifact. + let mut code = vec![tag::JUMP]; + code.extend_from_slice(&99u32.to_le_bytes()); + code.push(tag::RETURN); + + assert!(matches!( + check_bytes(code, 8), + Err(VerifyError::JumpOutOfRange { .. }), + )); + } + + /// A jump into an operand would read that operand's bytes as a tag, which + /// is how a byte-addressed chunk goes wrong in a way an index-addressed one + /// could not. + #[test] + fn rejects_a_jump_into_the_middle_of_an_instruction() { + let mut code = vec![tag::JUMP]; + code.extend_from_slice(&3u32.to_le_bytes()); // lands inside itself + code.push(tag::RETURN); + + assert_eq!( + check_bytes(code, 8), + Err(VerifyError::JumpIntoAnInstruction { at: 0, target: 3 }), + ); + } + + /// A switch is a jump with many targets, and every one of them needs the + /// proof an ordinary jump gets — otherwise the one arm nobody tested is + /// the one that decodes an operand as an opcode. + #[test] + fn every_arm_of_a_switch_is_checked() { + let ops = vec![ + Op::Unit, + Op::Switch(0), + Op::Unit, // index 2: the case arm + Op::Return, + Op::Unit, // index 4: the default + Op::Return, + ]; + let (code, offsets) = assemble(&ops).expect("must assemble"); + let chunk = Chunk::new(0, code.len() as u32, 8); + + let table = |case: u32, default: u32| Switch { + cases: vec![crate::grain::bytecode::SwitchCase { + hash: 7, + target: case, + }], + ranges: Vec::new(), + default, + }; + + let good = [table(offsets[2], offsets[4])]; + assert_eq!( + verify( + &code, + &[chunk], + Pools { + switches: &good, + ..pools() + } + ), + Ok(vec![1]), + ); + + // One byte into the `Switch` instruction's own operand. + let mid = [table(offsets[1] + 1, offsets[4])]; + assert!( + matches!( + verify( + &code, + &[chunk], + Pools { + switches: &mid, + ..pools() + } + ), + Err(VerifyError::JumpIntoAnInstruction { .. }), + ), + "a case arm landing mid-instruction must be refused", + ); + + // The default is a target like any other, and the easiest to forget. + let outside = [table(offsets[2], 9999)]; + assert!( + matches!( + verify( + &code, + &[chunk], + Pools { + switches: &outside, + ..pools() + } + ), + Err(VerifyError::JumpOutOfRange { .. }), + ), + "a default outside the chunk must be refused", + ); + } + + #[test] + fn rejects_popping_an_empty_stack() { + assert!(matches!( + check(vec![Op::Pop, Op::Return]), + Err(VerifyError::Underflow { .. }), + )); + } + + /// A rotate reaches under the operands above it, and one frame's operands + /// sit on the same stack as its caller's. Nothing at run time knows where + /// the frame started, so the depth it needs is checked here or nowhere. + #[test] + fn rejects_a_rotate_that_reaches_below_the_frame() { + assert!(matches!( + check(vec![Op::Unit, Op::Unit, Op::Rotate(2), Op::Return]), + Err(VerifyError::Underflow { + need: 3, + have: 2, + .. + }), + )); + assert_eq!( + check(vec![ + Op::Unit, + Op::Unit, + Op::Unit, + Op::Rotate(2), + Op::Return + ]), + Ok(vec![3]), + "with the third operand there it is in range, and nothing moves", + ); + } + + #[test] + fn rejects_running_past_the_last_instruction() { + assert_eq!(check(vec![Op::Unit]), Err(VerifyError::FallsOffTheEnd)); + } + + #[test] + fn rejects_an_index_with_nothing_behind_it() { + let (code, _) = assemble(&[Op::Const(7), Op::Return]).unwrap(); + let chunk = Chunk::new(0, code.len() as u32, 8); + assert!(matches!( + verify( + &code, + &[chunk], + Pools { + consts: 1, + ..pools() + } + ), + Err(VerifyError::BadIndex { + what: "constant", + .. + }), + )); + } + + #[test] + fn rejects_a_chunk_that_outgrows_its_declared_stack() { + let (code, _) = assemble(&[Op::Unit, Op::Unit, Op::Unit, Op::Return]).unwrap(); + assert!(matches!( + check_bytes(code, 2), + Err(VerifyError::StackExceedsDeclared { .. }), + )); + } + + /// Bytes that are not instructions must be named as such rather than + /// executed. + #[test] + fn rejects_a_tag_it_does_not_know() { + assert_eq!( + check_bytes(vec![0xff], 8), + Err(VerifyError::Undecodable { at: 0 }), + ); + } + + #[test] + fn rejects_an_instruction_whose_operands_are_cut_off() { + assert_eq!( + check_bytes(vec![tag::CONST, 0], 8), + Err(VerifyError::Undecodable { at: 0 }), + ); + } + + /// A chunk naming a span the code does not have is a corrupt artifact, not + /// a compiler bug — and must not index out of bounds. + #[test] + fn rejects_a_chunk_that_names_code_it_does_not_have() { + let (code, _) = assemble(&[Op::Unit, Op::Return]).unwrap(); + assert!(matches!( + verify(&code, &[Chunk::new(0, 9999, 8)], pools()), + Err(VerifyError::ChunkOutOfRange { .. }), + )); + } +} diff --git a/src/grain/compile/cases.rs b/src/grain/compile/cases.rs new file mode 100644 index 000000000..49a17af9d --- /dev/null +++ b/src/grain/compile/cases.rs @@ -0,0 +1,296 @@ +//! Turning rhai's `switch` range arms into something that can be looked up. +//! +//! Rhai scans its range arms in order and takes the first one that both +//! contains the subject *and* whose guard passes — so when two arms overlap, +//! which arm runs is not a property of the subject alone. A dispatch table has +//! one answer per subject, so the overlap has to go somewhere, and the only +//! place it can go without keeping the subject alive across the guards is +//! here. +//! +//! [`split`] cuts the arms into disjoint intervals. Each interval carries the +//! arms covering it, in source order, and the compiler emits those as a chain +//! of guards ending at the default — exactly what it already does for the arms +//! sharing a case value. + +use crate::{ast::RangeCase, INT}; + +use crate::grain::bytecode::SwitchRange; + +/// A range arm's interval, as bounds rather than as a `Range`. +fn bounds(range: &RangeCase) -> (INT, INT, bool) { + match range { + RangeCase::ExclusiveInt(r, ..) => (r.start, r.end, false), + RangeCase::InclusiveInt(r, ..) => (*r.start(), *r.end(), true), + } +} + +/// A piece of the number line that every arm either covers whole or not at +/// all. +/// +/// Two kinds because the bounds are integers but the values are not: between +/// two adjacent bounds there is nothing to enumerate and yet `5.5` is in +/// there, so the gaps are intervals in their own right rather than something +/// the endpoints cover. +enum Atom { + /// A single integer. + Point(INT), + /// Everything strictly between two adjacent bounds. + Between(INT, INT), +} + +fn covers(range: &RangeCase, atom: &Atom) -> bool { + match atom { + Atom::Point(point) => range.contains_int(*point), + // Every endpoint is a bound, so a range that reaches into a gap + // between two adjacent bounds spans the whole of it. + Atom::Between(low, high) => { + let (start, end, ..) = bounds(range); + start <= *low && end >= *high + } + } +} + +/// One interval and the arms that cover it, being built. +struct Run { + from: INT, + to: INT, + inclusive: bool, + blocks: Vec, +} + +/// Split range arms into disjoint intervals, each carrying the arms that cover +/// it in source order. +/// +/// Targets come back as zero: which instruction an interval sends control to +/// is the caller's to fill in, once it has emitted the chain. +/// +/// ## Why the lower bounds can overlap after all +/// +/// An interval that starts in a gap — everything above `10` but not `10` +/// itself — has no exact form here, and comes back as `10..hi`, which does +/// include `10`. That is sound because entries are scanned in order and `10` +/// is always claimed by an earlier one: any arm reaching into the gap above a +/// bound also covers the bound, so the point is never an interval nobody +/// emitted. +pub(crate) fn split(ranges: &[RangeCase]) -> Vec<(SwitchRange, Vec)> { + let mut points: Vec = Vec::with_capacity(ranges.len() * 2); + for range in ranges { + let (start, end, ..) = bounds(range); + points.push(start); + points.push(end); + } + points.sort_unstable(); + points.dedup(); + + let mut atoms = Vec::with_capacity(points.len() * 2); + for (index, point) in points.iter().enumerate() { + atoms.push(Atom::Point(*point)); + if let Some(next) = points.get(index + 1) { + atoms.push(Atom::Between(*point, *next)); + } + } + + let mut out: Vec<(SwitchRange, Vec)> = Vec::new(); + let mut run: Option = None; + + let finish = |run: Run| { + ( + SwitchRange { + from: run.from, + to: run.to, + inclusive: run.inclusive, + target: 0, + }, + run.blocks, + ) + }; + + for atom in &atoms { + let blocks: Vec = ranges + .iter() + .filter(|range| covers(range, atom)) + .map(RangeCase::index) + .collect(); + + let (from, to, inclusive) = match atom { + Atom::Point(point) => (*point, *point, true), + Atom::Between(low, high) => (*low, *high, false), + }; + + // Atoms are contiguous and in order, so one that runs the same arms as + // the one before it is the same table entry stretched further. + if run.as_ref().is_some_and(|open| open.blocks == blocks) { + let open = run.as_mut().expect("just checked"); + open.to = to; + open.inclusive = inclusive; + continue; + } + + if let Some(open) = run.take() { + out.push(finish(open)); + } + // An atom no arm covers ends the run and starts nothing: a subject + // there has no range arm and belongs to the default. + if !blocks.is_empty() { + run = Some(Run { + from, + to, + inclusive, + blocks, + }); + } + } + + if let Some(open) = run.take() { + out.push(finish(open)); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `from..to => block`, as rhai's parser would record it. + fn exclusive(from: INT, to: INT, block: usize) -> RangeCase { + let mut case: RangeCase = (from..to).into(); + case.set_index(block); + case + } + + fn inclusive(from: INT, to: INT, block: usize) -> RangeCase { + let mut case: RangeCase = (from..=to).into(); + case.set_index(block); + case + } + + /// What the table says, as `(from, to, inclusive, blocks)`. + fn table(ranges: &[RangeCase]) -> Vec<(INT, INT, bool, Vec)> { + split(ranges) + .into_iter() + .map(|(range, blocks)| (range.from, range.to, range.inclusive, blocks)) + .collect() + } + + #[test] + fn nothing_in_gives_nothing_out() { + assert!(split(&[]).is_empty()); + } + + /// The common shape. Splitting must not turn two arms into six entries, + /// or every artifact with a `switch` in it pays for the case nobody + /// wrote. + #[test] + fn arms_that_do_not_overlap_come_back_unchanged() { + assert_eq!( + table(&[inclusive(0, 9, 0), inclusive(10, 99, 1)]), + vec![(0, 9, true, vec![0]), (10, 99, true, vec![1])], + ); + } + + /// Two arms over the same values — the shape that has no single answer at + /// runtime, and the reason this exists. + #[test] + fn identical_arms_become_one_entry_running_both() { + assert_eq!( + table(&[inclusive(0, 9, 0), inclusive(0, 9, 1)]), + vec![(0, 9, true, vec![0, 1])], + ); + } + + #[test] + fn a_partial_overlap_splits_into_three() { + assert_eq!( + table(&[exclusive(0, 10, 0), exclusive(5, 20, 1)]), + vec![ + (0, 5, false, vec![0]), + (5, 10, false, vec![0, 1]), + (10, 20, false, vec![1]), + ], + ); + } + + /// The awkward one: the arms meet at a single integer that only one of + /// them includes, and the piece above it cannot be named exactly. + #[test] + fn an_inclusive_end_meeting_an_exclusive_start_splits_at_the_point() { + let split = table(&[inclusive(0, 10, 0), exclusive(10, 20, 1)]); + + assert_eq!( + split, + vec![ + (0, 10, false, vec![0]), + (10, 10, true, vec![0, 1]), + (10, 20, false, vec![1]), + ], + ); + + // The last entry does contain `10`, which belongs to both arms — the + // entry before it is what makes that unreachable, so the order is + // part of the answer and not a presentation detail. + let (shared, ..) = &split[1]; + assert!(*shared == 10, "the point entry must come first"); + } + + /// A hole between two arms is not an entry: a subject in it has no range + /// arm at all and must reach the default. + #[test] + fn a_gap_between_arms_is_left_out() { + assert_eq!( + table(&[inclusive(0, 4, 0), inclusive(10, 14, 1)]), + vec![(0, 4, true, vec![0]), (10, 14, true, vec![1])], + ); + } + + /// An arm inside another arm keeps the outer one either side of it. + #[test] + fn a_nested_arm_splits_the_one_around_it() { + assert_eq!( + table(&[exclusive(0, 100, 0), exclusive(10, 20, 1)]), + vec![ + (0, 10, false, vec![0]), + (10, 20, false, vec![0, 1]), + (20, 100, false, vec![0]), + ], + ); + } + + /// Splitting only means anything if what comes out agrees with rhai about + /// which arms a value belongs to — so check the pieces against the arms + /// they came from, at every bound and between them. + #[test] + fn every_value_reaches_the_arms_rhai_would_have_run() { + let arms = [ + exclusive(0, 10, 0), + inclusive(5, 20, 1), + exclusive(20, 25, 2), + inclusive(-5, 0, 3), + ]; + let split = split(&arms); + + let mut probes: Vec = Vec::new(); + for value in -8..=28 { + probes.push(rhai::Dynamic::from(value as INT)); + // Halfway between two integers is where an interval that has no + // exact form goes wrong, so the integers alone would not find it. + #[cfg(not(feature = "no_float"))] + probes.push(rhai::Dynamic::from(value as rhai::FLOAT + 0.5)); + } + + for probe in &probes { + let expected: Vec = arms + .iter() + .filter(|arm| arm.contains(probe)) + .map(RangeCase::index) + .collect(); + + let found = split + .iter() + .find(|(range, ..)| range.contains(probe)) + .map(|(.., blocks)| blocks.clone()) + .unwrap_or_default(); + + assert_eq!(found, expected, "for {probe:?}"); + } + } +} diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs new file mode 100644 index 000000000..4204e3d2f --- /dev/null +++ b/src/grain/compile/mod.rs @@ -0,0 +1,1996 @@ +mod cases; +mod poolable; +mod slots; + +use core::mem; + +use crate::ast::{ + ASTFlags, ASTNode, Expr, FlowControl, FnCallExpr, OpAssignment, ScriptFuncDef, Stmt, StmtBlock, + SwitchCasesCollection, +}; +use crate::tokenizer::Token; +use crate::types::Span; +use crate::{Dynamic, ImmutableString, Position, AST}; + +use crate::grain::bytecode::{ + assemble, resolve_switch_targets, AssignOp, Chain, Chunk, Op, Positions, Receiver, Root, Step, + Switch, SwitchCase, SwitchRange, Tail, +}; +use crate::grain::compile::poolable::is_poolable; +use crate::grain::compile::slots::Slots; +use crate::grain::program::{Function, Parts, Program}; + +/// Whether a variable reference is module-qualified, as in `foo::bar`. +/// +/// `Expr::Variable`'s payload only carries a `Namespace` when modules are +/// compiled in. Under `no_module` the box is two fields rather than four and +/// nothing can be qualified, so the question has a constant answer and the +/// field it would have read does not exist. +#[cfg(not(feature = "no_module"))] +macro_rules! has_namespace { + ($payload:expr) => { + !$payload.2.is_empty() + }; +} +#[cfg(feature = "no_module")] +macro_rules! has_namespace { + ($payload:expr) => {{ + let _ = $payload; + false + }}; +} + +/// The same question for a call: is it `foo::bar()` rather than `bar()`. +/// `FnCallExpr` carries no `namespace` field at all under `no_module`. +#[cfg(not(feature = "no_module"))] +macro_rules! call_has_namespace { + ($call:expr) => { + !$call.namespace.is_empty() + }; +} +#[cfg(feature = "no_module")] +macro_rules! call_has_namespace { + ($call:expr) => {{ + let _ = $call; + false + }}; +} + +/// Lowers a rhai `AST` into a [`Program`]. +/// +/// Anything not yet lowered is kept as an AST fragment and handed back to +/// rhai's walker at runtime, so the output always means the same as its input. +/// Progress is [`Program::residual_count`] falling. +#[derive(Debug, Default, Clone)] +pub struct Compiler { + _private: (), +} + +impl Compiler { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + #[must_use] + pub fn compile(&self, ast: &AST) -> Program<'static> { + // A bare script-function name used as a value is not a variable read + // at all — rhai turns it into a function pointer with the calling + // environment attached (`eval/expr.rs:71-99`) — so those names must + // not become `LoadNamed`. Carried across every restart below, because + // function bodies are lowered after one. + let script_fns: Vec = ast + .shared_lib() + .iter_script_fn_info() + .map(|(.., def)| def.name.clone()) + .collect(); + let fresh = || Lowering { + script_fns: script_fns.clone(), + ..Lowering::default() + }; + + let mut lowering = fresh(); + + // Anything the slot model cannot account for costs the whole program + // its lowering rather than risking a scope it resolved slots against + // being a different shape at runtime. Coverage is preserved either way. + if !lowering.program(ast.statements(), true) { + lowering = fresh(); + lowering.whole_program_residual(ast.statements()); + } + let main_ops = lowering.code.len(); + + // Each function's body appends to the same instruction list, so the + // whole program assembles as one address space. A function the slot + // model cannot handle is simply left out, and rhai's own copy of it + // stays reachable through the library below. + let mut functions = Vec::new(); + let mut skipped = 0usize; + for (.., def) in ast.shared_lib().iter_script_fn_info() { + match lowering.function(def) { + Some(function) => functions.push(function), + None => skipped += 1, + } + } + + // Assembly can fail the same way the slot model can, for a script with + // more distinct names or constants than a `u16` operand can index — so + // it takes the same exit. The fallback is a single instruction and + // always assembles, which is what keeps coverage total. + // Switch targets are instruction indices too, and they live in the + // pool rather than in the code, so they are resolved separately — + // failing the same way, into the same fallback. + let assembled = match assemble(&lowering.code) { + Ok((code, offsets)) => resolve_switch_targets(&mut lowering.switches, &offsets) + .ok() + .map(|()| (code, offsets)), + Err(..) => None, + }; + let (code, offsets, main_ops, functions, skipped) = match assembled { + Some((code, offsets)) => (code, offsets, main_ops, functions, skipped), + None => { + lowering = fresh(); + lowering.whole_program_residual(ast.statements()); + let (code, offsets) = + assemble(&lowering.code).expect("the fallback is one instruction"); + (code, offsets, lowering.code.len(), Vec::new(), 1) + } + }; + + // Jump targets and the position table were both keyed on instruction + // index while lowering; instructions vary in length once assembled. + let mut positions = vec![rhai::Position::NONE; code.len()]; + for (index, pos) in lowering.positions.iter().enumerate() { + positions[offsets[index] as usize] = *pos; + } + + let main = Chunk::new(0, offsets[main_ops], lowering.max_stack); + let functions: Vec<_> = functions + .into_iter() + .map(|f| Function { + name: f.name, + params: f.params, + chunk: Chunk::new( + offsets[f.first_op], + offsets[f.first_op + f.op_count], + lowering.max_stack, + ), + }) + .collect(); + + // rhai's own functions are carried whenever anything might still reach + // for them: a function this compiler skipped, or a fragment that could + // call one. With neither, every call resolves in the table above and + // the library — an `AST`'s whole function tree — can be dropped. + let needs_walker = skipped > 0 || !lowering.residuals.is_empty(); + + let mut program = Program::new( + code.into(), + main, + functions, + Parts { + positions: Positions::dense(positions), + residuals: lowering.residuals, + consts: lowering.consts, + names: crate::grain::bytecode::Strings::new(&lowering.names), + tokens: lowering.tokens, + assign_ops: lowering.assign_ops, + chains: lowering.chains, + switches: lowering.switches, + lib: (needs_walker && !ast.shared_lib().is_empty()) + .then(|| ast.shared_lib().clone()), + #[cfg(not(feature = "no_module"))] + resolver: ast.resolver.clone(), + source: ast.source().map(Into::into), + }, + ); + + // `max_stack` above is an upper bound the lowering can compute without + // a depth walk. The verifier does the walk anyway, so take its answer. + program.tighten_stack(); + program + } +} + +/// Where `break` and `continue` jump to, and what they must unwind first. +/// +/// Jump targets are backpatched: `break` sites are collected as they are +/// emitted and pointed at the instruction after the loop once that address is +/// known. +struct Loop { + /// Where `continue` goes — the condition test, or the top of the body. + continue_target: u32, + /// Slot depth a `break` unwinds to. For a `for` loop this is *before* the + /// loop variable, which leaving must drop. + break_depth: u16, + /// Slot depth a `continue` unwinds to. Differs from `break_depth` in a + /// `for`, where the loop variable has to survive into the next iteration — + /// one field cannot be both. + continue_depth: u16, + /// How many iterators are live *inside* this loop, so a jump out of it + /// can drop whatever was made since. A `break` inside a `try` inside a + /// `for` skips the straight-line path that would have cleaned up. + iters: usize, + /// Whether the loop owns an iterator of its own. `break` drops it and + /// `continue` must not, which is the other thing one field cannot be. + owns_iterator: bool, + /// How many `try` regions were armed when the loop began, so a jump out + /// of the loop disarms the ones inside it. + handlers: usize, + /// `Jump` sites awaiting the address after the loop. + breaks: Vec, +} + +/// Where a `switch` table entry sends control, before the arms have +/// addresses. +#[derive(Debug, Clone, Copy)] +enum Entry { + /// Straight to an arm's body: the group has no guard to try first. + Body(usize), + /// The head of a guard chain, which is already emitted. + At(u32), + /// Nothing in the group can run. + Default, +} + +/// A function body that lowered, before its instruction indices become byte +/// addresses. +struct LoweredFn { + name: u32, + params: Vec, + first_op: usize, + op_count: usize, +} + +#[derive(Default)] +struct Lowering { + code: Vec, + /// One per instruction, parallel to `code`. Most are `NONE`; the dense + /// shape is what makes a lookup an index, and it compacts on the way out. + positions: Vec, + residuals: Vec, + consts: Vec, + names: Vec, + tokens: Vec, + assign_ops: Vec, + chains: Vec, + switches: Vec, + slots: Slots, + max_stack: u16, + loops: Vec, + /// How many iterators are live at this point in the lowering, so a jump + /// out of a loop knows how many to drop. + iters: usize, + /// The same for `try` regions: a `break` out of one has to disarm it, or + /// the next unrelated error is caught into a block already left. + handlers: usize, + /// Names that are script functions rather than variables. + script_fns: Vec, + /// Set when something nested inside an expression defeated the slot model. + /// + /// [`Lowering::statement`] says so by returning false, but + /// [`Lowering::expression`] has no way to: it is called from the middle of + /// building other expressions, and every one of those callers would have + /// to thread the answer back. So a block used as an expression records the + /// failure here instead, and [`Lowering::program`] reports it. + /// + /// A sticky flag is enough because failure is all or nothing — the caller + /// throws the whole lowering away and starts again as one fragment — so + /// instructions emitted after it are discarded rather than run. + defeated: bool, +} + +impl Lowering { + /// Lower a statement list as a whole chunk. Returns false if something + /// defeated the slot model and the caller should fall back. + /// + /// `keeps_scope` says whether what this chunk declares outlives it, which + /// is true of the program and false of every function body. Only then is a + /// [`Op::Checkpoint`] worth emitting: it is what an escaping error unwinds + /// to, and a function's scope is discarded whole however it ends. + fn program(&mut self, statements: &[Stmt], keeps_scope: bool) -> bool { + let Some((last, leading)) = statements.split_last() else { + self.emit(Op::Unit); + self.emit(Op::Return); + return true; + }; + + for stmt in leading { + if keeps_scope { + self.emit(Op::Checkpoint); + } + if !self.statement(stmt) { + return false; + } + // A statement's value is only the program's value if it is the + // last one; rhai discards the rest. + self.emit(Op::Pop); + } + + if keeps_scope { + self.emit(Op::Checkpoint); + } + if !self.statement(last) { + return false; + } + + self.emit(Op::Return); + !self.defeated + } + + /// Lower `a.b[i].c`, either reading it or assigning to it. + /// + /// Returns false if the chain is not one this can express, in which case + /// the caller keeps it as a fragment. + /// + /// The shape is the awkward part. Rhai does not store a chain as a list: + /// `a.b[i]` is `Dot { lhs: a, rhs: Index { lhs: b, rhs: i } }`, where each + /// nested node's `lhs` is the *current* step's operand and its `rhs` is the + /// continuation. [`flatten_chain`] unpicks that into steps. + fn chain(&mut self, expr: &Expr, tail: Tail, value: Option<&Expr>) -> bool { + let Some((root, steps)) = flatten_chain(expr) else { + return false; + }; + + // A variable root is one the chain can write back into, by slot or by + // name; anything else has to be both a read and a value rhai would + // itself have evaluated into a temporary. + // + // `this` is deliberately not in the second class. Rhai reaches it + // through the caller's `&mut` (`eval/chaining.rs:528`), so a method + // step that mutates lands in the caller's value — walking a copy would + // drop the write silently, which is the one failure mode worth + // refusing outright. It is also unreachable: a body that mentions + // `this` is never compiled to a chunk at all. + let root_spec = match root { + Expr::Variable(v, ..) if !has_namespace!(v) => match self.slots.resolve(&v.1) { + Some(slot) => Root::Local { + slot, + name: self.push_name(v.1.clone()), + }, + // The caller's, or a module's, or nothing — decided at run + // time, because which of the three it is decides whether the + // chain can write through it. + // + // The guard is load-bearing: a bare script-function name is a + // function pointer rather than a variable, and turning one + // into a name lookup would report it missing where rhai hands + // back a pointer. + None if self.is_variable_name(&v.1, false) => Root::Named { + name: self.push_name(v.1.clone()), + pos: root.position(), + }, + None => return false, + }, + // A qualified root resolves against imported modules, which need + // `import` — the escape hatch's job. + Expr::Variable(..) | Expr::ThisPtr(..) => return false, + _ if matches!(tail, Tail::Read) => Root::Temporary, + // Unreachable through the parser, which refuses `f().x = 1` + // outright (`eval/chaining.rs:559`). + _ => return false, + }; + + // Index values and method arguments are evaluated first, in step + // order, exactly as rhai collects them before walking + // (`eval/chaining.rs:568`). Evaluating one partway down would need the + // operand stack while a borrow of the container is live. + let mut lowered = Vec::with_capacity(steps.len()); + let mut operands = 0u16; + + for step in &steps { + match step { + ChainStep::Index(index, bracket) => { + self.expression(index); + lowered.push(Step::Index { + operand: operands, + pos: index.start_position(), + bracket: *bracket, + }); + operands += 1; + } + ChainStep::Property(prop, pos) => { + let (getter, setter, name) = &**prop; + lowered.push(Step::Property { + name: self.push_name(name.clone()), + getter: self.push_name(getter.0.clone()), + setter: self.push_name(setter.0.clone()), + pos: *pos, + }); + } + ChainStep::Method(call, pos) => { + if !self.is_lowerable_call(call) { + return false; + } + let Ok(argc) = u8::try_from(call.args.len()) else { + return false; + }; + let first = operands; + for arg in call.args.iter() { + self.expression(arg); + operands += 1; + } + lowered.push(Step::Method { + name: self.push_name(call.name.clone()), + argc, + operand: first, + pos: *pos, + }); + } + } + } + + // Then the root, if it is one that has to be evaluated. After the + // operands rather than before, which is rhai's order and not the + // reading order: `[f()][g()]` calls `g` first. + if matches!(root_spec, Root::Temporary) { + self.expression(root); + } + + // The value being assigned goes on last, above everything, so the + // walk can take what it needs before it borrows the container. + if let Some(value) = value { + self.expression(value); + } + + let index = self.push_chain(Chain { + root: root_spec, + steps: lowered, + tail, + operands, + }); + self.emit_at(Op::Chain(index), expr.position()); + true + } + + /// Lower a `switch` into one dispatch table plus the arms it names. + /// + /// The layout is: the subject, [`Op::Switch`], the guard chains, the arm + /// bodies, the default. Every arm leaves one value and jumps to the end, + /// so the statement's value is the matched arm's — or unit, which is what + /// an absent `_` compiles to. + /// + /// Guards are why the table does not simply hold bodies. Rhai tries the + /// arms sharing a case value in source order and falls to the *default* + /// when they all decline — never on to the ranges (`eval/stmt.rs:544`) — + /// so a group becomes a chain of guards ending in a jump to the default, + /// and the table points at the chain. Nearly every arm anyone writes has + /// no guard, and those cost no chain at all. + fn switch(&mut self, subject: &Expr, sw: &SwitchCasesCollection) -> bool { + // Sorted because rhai's map iterates in whatever order its hasher put + // the entries in, and an artifact should not depend on that. + let mut groups: Vec<(u64, Vec)> = sw + .cases + .iter() + .map(|(hash, blocks)| (*hash, blocks.to_vec())) + .collect(); + groups.sort_unstable_by_key(|(hash, ..)| *hash); + + // Overlapping range arms have no single answer at runtime, so they are + // cut into disjoint pieces here instead. See [`cases::split`]. + let ranges = cases::split(&sw.ranges); + + // Unflattened: a shared subject is not hashable, and rhai's gate on + // that is what sends it to the default arm. Flattening here would make + // it match a case the walker skips. + self.unflattened(subject); + let table = self.push_switch(); + self.emit(Op::Switch(table)); + + // One chain per distinct list of arms, shared by every table entry + // naming it: `1 | 2 => ..` is two case values and one chain. + let mut chains: Vec<(&[usize], Entry)> = Vec::new(); + let mut to_body: Vec<(usize, usize)> = Vec::new(); + let mut to_default: Vec = Vec::new(); + + let lists = groups + .iter() + .map(|(.., blocks)| blocks.as_slice()) + .chain(ranges.iter().map(|(.., blocks)| blocks.as_slice())); + for blocks in lists { + if chains.iter().any(|(existing, ..)| *existing == blocks) { + continue; + } + let entry = self.arm_chain(sw, blocks, &mut to_body, &mut to_default); + chains.push((blocks, entry)); + } + + // Bodies, one per arm something can reach. An arm behind a constant + // false guard, or one whose range the parser dropped for being empty, + // is reachable by nothing and is not emitted. + let mut wanted: Vec = to_body.iter().map(|(.., block)| *block).collect(); + wanted.extend(chains.iter().filter_map(|(.., entry)| match entry { + Entry::Body(block) => Some(*block), + _ => None, + })); + wanted.extend(sw.def_case); + wanted.sort_unstable(); + wanted.dedup(); + + let mut body_at: Vec<(usize, u32)> = Vec::with_capacity(wanted.len()); + let mut to_end: Vec = Vec::with_capacity(wanted.len()); + for block in wanted { + body_at.push((block, self.here())); + // An arm body is an ordinary expression, and a block one goes + // through the same path as `let y = { .. }`. + self.expression(&sw.expressions[block].rhs); + if self.defeated { + return false; + } + to_end.push(self.emit_jump()); + } + + let at = |block: usize| { + body_at + .iter() + .find(|(candidate, ..)| *candidate == block) + .map(|(.., at)| *at) + .expect("every reachable arm was emitted above") + }; + + let default_at = match sw.def_case { + Some(block) => at(block), + None => { + let target = self.here(); + self.emit(Op::Unit); + target + } + }; + let end = self.here(); + + for site in to_end { + self.patch_to(site, end); + } + for site in to_default { + self.patch_to(site, default_at); + } + for (site, block) in to_body { + self.patch_to(site, at(block)); + } + + let target = |blocks: &[usize]| { + let entry = chains + .iter() + .find(|(existing, ..)| *existing == blocks) + .map(|(.., entry)| *entry) + .expect("every list got a chain above"); + match entry { + Entry::Body(block) => at(block), + Entry::At(target) => target, + Entry::Default => default_at, + } + }; + + self.switches[table as usize] = Switch { + cases: groups + .iter() + .map(|(hash, blocks)| SwitchCase { + hash: *hash, + target: target(blocks), + }) + .collect(), + ranges: ranges + .iter() + .map(|(range, blocks)| SwitchRange { + target: target(blocks), + ..*range + }) + .collect(), + default: default_at, + }; + + true + } + + /// Emit the guard chain for one group of arms, and say where the table + /// entries naming that group should point. + fn arm_chain( + &mut self, + sw: &SwitchCasesCollection, + blocks: &[usize], + to_body: &mut Vec<(usize, usize)>, + to_default: &mut Vec, + ) -> Entry { + let mut entry: Option = None; + + for block in blocks { + match &sw.expressions[*block].lhs { + // An arm without an `if` is a literal `true` in the tree + // (`parser.rs:1187`), so it always runs and everything after + // it in the group is unreachable. + Expr::BoolConstant(true, ..) => { + return match entry { + None => Entry::Body(*block), + Some(entry) => { + to_body.push((self.emit_jump(), *block)); + entry + } + }; + } + // Nothing can reach this arm, so nothing is emitted for it. + Expr::BoolConstant(false, ..) => continue, + guard => { + if entry.is_none() { + entry = Some(Entry::At(self.here())); + } + self.expression(guard); + let site = self.code.len(); + // Rhai reports a non-boolean guard against the guard, so + // the jump carries the guard's position. + self.emit_at(Op::JumpIfTrue { target: u32::MAX }, guard.position()); + to_body.push((site, *block)); + } + } + } + + match entry { + Some(entry) => { + to_default.push(self.emit_jump()); + entry + } + // Every arm in the group is behind a constant false guard, so the + // group is the default with extra steps. + None => Entry::Default, + } + } + + /// Reserve a table, to be filled in once its arms have addresses. + fn push_switch(&mut self) -> u32 { + self.switches.push(Switch { + cases: Vec::new(), + ranges: Vec::new(), + default: 0, + }); + (self.switches.len() - 1) as u32 + } + + fn push_chain(&mut self, chain: Chain) -> u32 { + if let Some(index) = self.chains.iter().position(|existing| *existing == chain) { + return index as u32; + } + self.chains.push(chain); + (self.chains.len() - 1) as u32 + } + + /// Lower one script function's body into the same instruction list. + /// + /// Returns `None` if the slot model cannot account for it, in which case + /// rhai keeps its own copy and calls to it go through dispatch. That is a + /// per-function decision: one awkward function does not cost the rest + /// their lowering. + /// + /// The body runs in a fresh scope with the parameters already pushed + /// (`func/script.rs:73`), so the parameters are exactly slots 0 upwards. + fn function(&mut self, def: &ScriptFuncDef) -> Option { + // `this` is not a scope entry, so no slot addresses it. The declared + // type is not enough to go on: an untyped `fn double() { this * 2 }` + // leaves `this_type` empty and still needs one, so the body is what + // has to be checked. + if def.this_type.is_some() || uses_this(def.body.statements()) { + return None; + } + + let first_op = self.code.len(); + let saved_slots = mem::take(&mut self.slots); + let saved_loops = mem::take(&mut self.loops); + // Per-function, like the slots: one body the model cannot handle must + // not cost the rest of the program its lowering. + let saved_defeated = mem::replace(&mut self.defeated, false); + + for param in def.params.iter() { + self.slots.declare(param.clone()); + } + let params: Vec<_> = def + .params + .iter() + .map(|p| self.push_name(p.clone())) + .collect(); + + // A body is a statement list whose last value is the return value, + // which is what `program` already does. + let lowered = self.program(def.body.statements(), false); + + self.slots = saved_slots; + self.loops = saved_loops; + self.defeated = saved_defeated; + + if !lowered { + // Roll back whatever the attempt emitted, so a function that could + // not be lowered leaves no unreachable instructions behind. + self.code.truncate(first_op); + self.positions.truncate(first_op); + return None; + } + + Some(LoweredFn { + name: self.push_name(def.name.clone()), + params, + first_op, + op_count: self.code.len() - first_op, + }) + } + + /// The M0 fallback: one fragment holding everything, evaluated without + /// rewinding so top-level declarations still reach the caller. + fn whole_program_residual(&mut self, statements: &[Stmt]) { + let body = wrap_statements(statements.to_vec()); + let residual = self.push_residual(body); + self.emit(Op::EvalAst { + residual, + rewind_scope: false, + }); + self.emit(Op::Return); + } + + /// Lower one statement, leaving its value on the stack. + fn statement(&mut self, stmt: &Stmt) -> bool { + match stmt { + Stmt::Var(payload, flags, ..) => { + // `export let x = ...` also binds a module alias, which the + // slot model does not represent. + if flags.intersects(ASTFlags::EXPORTED) || self.slots.is_full() { + return false; + } + + let (ident, init, ..) = &**payload; + self.expression(init); + + let name = self.push_name(ident.name.clone()); + self.slots.declare(ident.name.clone()); + self.emit(Op::DeclareLocal { + name, + is_const: flags.intersects(ASTFlags::CONSTANT), + }); + + // A declaration evaluates to unit. + self.emit(Op::Unit); + true + } + + Stmt::Expr(expr) => { + self.expression(expr); + true + } + + // Rhai gives a call standing alone as a statement its own node + // rather than wrapping it in `Stmt::Expr`, and an operator is a + // call — so without this every top-level `a * b` stayed a fragment. + // A closure's `curry` lands here rather than in `Stmt::Expr`, + // because rhai gives a call standing alone as a statement its own + // node. + Stmt::FnCall(call, pos) if self.fn_ptr_call(call, *pos) => true, + + Stmt::FnCall(call, pos) if self.is_lowerable_call(call) => { + self.lower_call(call, *pos); + true + } + + // A plain local on the left. + Stmt::Assignment(payload) + if matches!(&payload.1.lhs, Expr::Variable(v, ..) + if !has_namespace!(v) && self.slots.resolve(&v.1).is_some()) => + { + let (op_info, binary) = &**payload; + let Expr::Variable(v, ..) = &binary.lhs else { + unreachable!("checked by the guard"); + }; + let slot = self.slots.resolve(&v.1).expect("checked by the guard"); + let var_name = self.push_name(v.1.clone()); + + self.expression(&binary.rhs); + let op = self.op_assignment(op_info); + + self.emit_at(Op::AssignLocal { slot, var_name, op }, op_info.position()); + self.emit(Op::Unit); + true + } + + // A variable no slot names — the caller's. Same shape as above, + // and the same op-assignment resolution; only where the target + // lives differs. + Stmt::Assignment(payload) + if matches!(&payload.1.lhs, Expr::Variable(v, ..) + if self.is_variable_name(&v.1, has_namespace!(v))) => + { + let (op_info, binary) = &**payload; + let Expr::Variable(v, ..) = &binary.lhs else { + unreachable!("checked by the guard"); + }; + let name = self.push_name(v.1.clone()); + + self.expression(&binary.rhs); + let op = self.op_assignment(op_info); + + // The variable's position, not the operator's — unlike + // `AssignLocal`. The errors this instruction raises itself are + // `ErrorAssignmentToConstant` and `ErrorVariableNotFound`, and + // rhai reports both against the variable (`eval/stmt.rs:340` + // and `eval/stmt.rs:120`). For a local those are unreachable, + // because the parser rejects a constant it can see; for a name + // the caller supplied they are the common failures. + self.emit_at(Op::AssignNamed { name, op }, binary.lhs.position()); + self.emit(Op::Unit); + true + } + + // A chain on the left. The value goes on the stack after the + // chain's own operands, so the walk has everything it needs before + // it takes a borrow of the container. + Stmt::Assignment(payload) + if matches!(&payload.1.lhs, Expr::Dot(..) | Expr::Index(..)) => + { + let (op_info, binary) = &**payload; + let op = self.op_assignment(op_info); + + let mark = self.mark(); + if !self.chain(&binary.lhs, Tail::Assign { op }, Some(&binary.rhs)) { + self.rewind(mark); + let residual = self.push_residual(wrap_statements(vec![stmt.clone()])); + self.emit(Op::EvalAst { + residual, + rewind_scope: true, + }); + } + // The chain leaves unit, which is what an assignment evaluates + // to, so there is nothing to add here. + true + } + + // Emitted by the parser ahead of the `curry` call that binds a + // closure's captures (`parser.rs:3707`). + #[cfg(not(feature = "no_closure"))] + Stmt::Share(names) => { + for (ident, ..) in names.iter() { + match self.slots.resolve(&ident.name) { + Some(slot) => self.emit_at(Op::Share(slot), ident.pos), + None => { + // The caller's — a closure can capture something + // no slot addresses. + let name = self.push_name(ident.name.clone()); + self.emit_at(Op::ShareNamed(name), ident.pos); + } + } + } + self.emit(Op::Unit); + true + } + + Stmt::Block(block) => self.block(block.statements()), + + // `try { .. } catch (e) { .. }`. + // + // The catch block's value is thrown away: rhai's whole statement + // is the try block's value on the way through and *unit* when + // something was caught (`.map(|_| Dynamic::UNIT)`, + // `eval/stmt.rs:863`). So `try { throw 7 } catch (e) { e * 2 }` is + // unit, not 14. + Stmt::TryCatch(payload, ..) => { + let FlowControl { expr, body, branch } = &**payload; + + // An absent catch variable is `Expr::Unit`; a present one is + // an `Expr::Variable` whose position is what rhai reports + // `ErrorTooManyVariables` against. + let catch_var = match expr { + Expr::Variable(v, ..) => Some(v.1.clone()), + _ => None, + }; + + let catch_name = catch_var.clone().map(|name| self.push_name(name)); + let site = self.code.len(); + self.emit_at( + Op::PushHandler { + target: u32::MAX, + catch_var: catch_name, + }, + expr.position(), + ); + self.handlers += 1; + + if !self.block(body.statements()) { + return false; + } + self.emit(Op::PopHandler); + self.handlers -= 1; + let past = self.emit_jump(); + + // The catch block, entered with the scope back where the `try` + // began and the variable already pushed on top of it. The + // handler is still armed here — that is what makes a bare + // `throw;` in this block a re-raise — so the depth goes back + // up, and the `PopHandler` below is what ends the region. + self.patch_to(site, self.here()); + self.handlers += 1; + let depth = self.slots.depth(); + if let Some(name) = catch_var { + self.slots.declare(name); + } + if !self.block(branch.statements()) { + return false; + } + self.emit(Op::Pop); + self.unwind_to(depth); + self.emit(Op::PopHandler); + self.handlers -= 1; + self.emit(Op::Unit); + + self.patch_here(past); + true + } + + // `for x in seq` / `for (x, i) in seq`. + // + // The loop variable and counter are pushed once and written each + // time round, not re-pushed — rhai does the same (`stmt.rs:708`), + // and it is observable: a closure made in the body captures the + // cell, so every one of them sees the last value. + Stmt::For(payload, ..) => { + let (var, counter, flow) = &**payload; + let outside = u16::try_from(self.slots.depth()).expect("slot count is bounded"); + + self.expression(&flow.expr); + // `ErrorFor` is reported against the iterable's *start*, which + // for `a.b` or a call is not its `position`. + self.emit_at(Op::IterInit, flow.expr.start_position()); + self.iters += 1; + + // Counter first, matching the order rhai pushes them in, so + // the slots line up with the scope it builds. + let counter_slot = counter.as_ref().map(|ident| { + let name = self.push_name(ident.name.clone()); + self.emit(Op::Unit); + self.emit(Op::DeclareLocal { + name, + is_const: false, + }); + self.slots.declare(ident.name.clone()); + self.slots.depth() as u16 - 1 + }); + let var_name = self.push_name(var.name.clone()); + self.emit(Op::Unit); + self.emit(Op::DeclareLocal { + name: var_name, + is_const: false, + }); + self.slots.declare(var.name.clone()); + let var_slot = self.slots.depth() as u16 - 1; + + let top = self.here(); + let exit = self.code.len(); + self.emit_at( + Op::IterNext { + exit: u32::MAX, + indexed: counter_slot.is_some(), + }, + flow.expr.position(), + ); + // The item is on top, the count under it, so these pop in the + // order the two locals were declared. + self.emit(Op::StoreShared(var_slot)); + if let Some(slot) = counter_slot { + self.emit(Op::StoreShared(slot)); + } + self.emit_at(Op::Tick, flow.body.position()); + + self.begin_for(top, outside); + if !self.block_discarding(flow.body.statements()) { + return false; + } + self.emit(Op::Jump(top)); + let breaks = self.end_loop(); + + // Exhausted: `IterNext` dropped the iterator on the way here. + self.patch_to(exit, self.here()); + self.iters -= 1; + self.emit(Op::UnwindTo(outside)); + self.slots.unwind_to(outside as usize); + + self.emit(Op::Unit); + let past = self.emit_jump(); + for site in breaks { + self.patch_here(site); + } + self.patch_here(past); + true + } + + Stmt::Switch(payload, ..) => { + let (subject, cases) = &**payload; + self.switch(subject, cases) + } + + Stmt::If(payload, ..) => { + let FlowControl { expr, body, branch } = &**payload; + + self.expression(expr); + let to_else = self.emit_jump_if_false(expr.position()); + + if !self.block(body.statements()) { + return false; + } + let past_else = self.emit_jump(); + + self.patch_here(to_else); + if !self.block(branch.statements()) { + return false; + } + self.patch_here(past_else); + true + } + + // `loop` and `while true` are the same node: rhai marks an + // unconditional loop with a unit or `true` guard + // (`eval/stmt.rs:575-576`). + Stmt::While(payload, ..) => { + let FlowControl { expr, body, .. } = &**payload; + let unconditional = matches!(expr, Expr::Unit(..) | Expr::BoolConstant(true, ..)); + + let top = self.here(); + self.emit_at(Op::Tick, body.position()); + + let exit = if unconditional { + None + } else { + self.expression(expr); + Some(self.emit_jump_if_false(expr.position())) + }; + + self.begin_loop(top); + if !self.block_discarding(body.statements()) { + return false; + } + self.emit(Op::Jump(top)); + + let breaks = self.end_loop(); + if let Some(exit) = exit { + self.patch_here(exit); + } + // A `while` that runs to completion is unit; a `break value` + // supplies its own. Both arrive here with the stack balanced. + self.emit(Op::Unit); + let past = self.emit_jump(); + for site in breaks { + self.patch_here(site); + } + self.patch_here(past); + true + } + + Stmt::Do(payload, flags, ..) => { + let FlowControl { expr, body, .. } = &**payload; + let until = flags.intersects(ASTFlags::NEGATED); + + let top = self.here(); + self.emit_at(Op::Tick, body.position()); + + self.begin_loop(top); + if !self.block_discarding(body.statements()) { + return false; + } + let breaks = self.end_loop(); + + self.expression(expr); + if until { + // `do ... until c` loops while `c` is false, which is a + // false-jump straight back to the top. + self.emit_at(Op::JumpIfFalse { target: top }, expr.position()); + } else { + let exit = self.emit_jump_if_false(expr.position()); + self.emit(Op::Jump(top)); + self.patch_here(exit); + } + + self.emit(Op::Unit); + let past = self.emit_jump(); + for site in breaks { + self.patch_here(site); + } + self.patch_here(past); + true + } + + Stmt::BreakLoop(value, flags, ..) => { + let Some(active) = self.loops.last() else { + // Outside any loop this is a parse error in rhai, so it + // should be unreachable; bail rather than emit a jump to + // nowhere. + return false; + }; + let continue_target = active.continue_target; + let loop_iters = active.iters; + let loop_handlers = active.handlers; + let owns_iterator = active.owns_iterator; + let (break_depth, continue_depth) = (active.break_depth, active.continue_depth); + let is_break = flags.intersects(ASTFlags::BREAK); + + // A jump out of a loop skips whatever the straight-line path + // would have cleaned up. The nesting is lexical, so how many + // iterators are live is known here — a `break` inside a `try` + // inside a `for` has one to drop, and `continue` has none + // because it re-enters the loop that owns it. + if is_break { + match value { + Some(expr) => self.expression(expr), + None => self.emit(Op::Unit), + } + // Out of the loop entirely, so its own iterator goes too — + // `loop_iters` counts from inside the loop and therefore + // already includes it. + self.pop_handlers(loop_handlers); + self.drop_iterators(loop_iters - usize::from(owns_iterator)); + self.emit(Op::UnwindTo(break_depth)); + let site = self.emit_jump(); + self.loops.last_mut().expect("checked").breaks.push(site); + } else { + // Back into the same loop, so its iterator and its loop + // variable both have to survive. + self.pop_handlers(loop_handlers); + self.drop_iterators(loop_iters); + self.emit(Op::UnwindTo(continue_depth)); + self.emit(Op::Jump(continue_target)); + } + + // Unreachable, but every statement must leave a value for the + // caller's `Pop`, and the verifier checks depth on every path. + self.emit(Op::Unit); + true + } + + // `throw` shares this node, flagged, and unwinds as an error + // rather than returning. The position is the keyword's, not the + // expression's (`eval/stmt.rs:877`). + Stmt::Return(value, flags, pos) if flags.intersects(ASTFlags::BREAK) => { + match value { + Some(expr) => self.expression(expr), + None => self.emit(Op::Unit), + } + self.emit_at(Op::Throw, *pos); + // Unreachable, but every statement leaves a value for the + // caller's `Pop` and the verifier checks depth on every path. + self.emit(Op::Unit); + true + } + + Stmt::Return(value, flags, ..) if !flags.intersects(ASTFlags::BREAK) => { + match value { + Some(expr) => self.expression(expr), + None => self.emit(Op::Unit), + } + self.emit(Op::Return); + self.emit(Op::Unit); + true + } + + // Not lowered yet. Safe as a fragment because every remaining + // statement kind either declares nothing or rewinds what it + // declares, so the scope is the same shape afterwards. + other => { + let residual = self.push_residual(wrap_statements(vec![other.clone()])); + self.emit(Op::EvalAst { + residual, + rewind_scope: true, + }); + true + } + } + } + + /// Lower one expression, leaving its value on the stack. + fn expression(&mut self, expr: &Expr) { + match expr { + Expr::BoolConstant(value, ..) => self.emit(Op::Bool(*value)), + Expr::Unit(..) => self.emit(Op::Unit), + + Expr::IntegerConstant(value, ..) => self.constant(Dynamic::from(*value)), + Expr::CharConstant(value, ..) => self.constant(Dynamic::from(*value)), + Expr::StringConstant(value, ..) => self.constant(Dynamic::from(value.clone())), + Expr::FloatConstant(value, ..) => self.constant(Dynamic::from(**value)), + // Folded by the optimizer, so it can hold anything a constant call + // returned — including a function pointer, which must not be + // copied out of a pool. See `poolable`. + // A function pointer the optimizer folded — `Fn("f")` with a + // constant name, or a closure literal. It cannot go in the pool: + // a closure's carries a `ScriptFuncDef`, which is an AST body and + // exactly what an artifact must not contain. Rebuilt by name + // instead, which reaches the chunk we compiled from that same + // body. + // + // A constant function pointer: a closure literal, or what the + // optimizer folds `Fn("f")` into. Either way it embeds a + // `ScriptFuncDef` — an AST body, `Fn*` in rhai's own rendering — + // so it cannot go in the pool. Rebuilt by name instead, reaching + // the chunk compiled from that same body. + // + // There is no version of this that keeps the rendering: the thing + // that differs *is* the tree, and carrying it is what an artifact + // must not do. `a_closure_pointer_is_late_bound` pins the + // difference for both spellings. + // + // Curried values are arbitrary `Dynamic`s with the same problem one + // level down, and are left to the walker. + Expr::DynamicConstant(value, ..) + if value + .read_lock::() + .is_some_and(|f| f.curry().is_empty()) => + { + let name = value + .read_lock::() + .expect("checked by the guard") + .fn_name() + .to_string(); + let name = self.push_name(name.into()); + self.emit_at(Op::MakeClosure(name), expr.position()); + } + + Expr::DynamicConstant(value, ..) if is_poolable(value) => { + self.constant((**value).clone()); + } + + Expr::Variable(payload, ..) => { + // A qualified name resolves against imported modules, not the + // scope, so it is not a slot. + let is_qualified = has_namespace!(payload); + + match self.slots.resolve(&payload.1) { + Some(slot) if !is_qualified => self.emit(Op::LoadLocal(slot)), + // Not a local this compiler declared, so no slot can name + // it: it is the caller's, a module's, or nothing. Looked + // up by name at run time, at the cost of a scope scan. + _ if self.is_variable_name(&payload.1, is_qualified) => { + let name = self.push_name(payload.1.clone()); + self.emit_at(Op::LoadNamed(name), expr.position()); + } + // A qualified name resolves against imported modules, and + // a bare function name is a function pointer. Neither is a + // variable read, and both stay rhai's job. + _ => self.residual_expr(expr), + } + } + + Expr::And(operands, ..) => self.short_circuit(operands, false), + Expr::Or(operands, ..) => self.short_circuit(operands, true), + + Expr::FnCall(call, pos) if self.fn_ptr_call(call, *pos) => {} + + Expr::FnCall(call, pos) if self.is_lowerable_call(call) => { + self.lower_call(call, *pos); + } + + // A literal whose elements are all constant never reaches here — + // rhai's optimizer folds it into a `DynamicConstant` first — so + // this is the one that has to be built at run time. + Expr::Array(elements, ..) if elements.len() <= u16::MAX as usize => { + for (index, element) in elements.iter().enumerate() { + self.expression(element); + // Positioned at the element, because that is what rhai + // blames when this element is the one that tips the + // running total over the limit. + self.emit_at( + Op::CheckSize { + index: index as u16, + map: false, + }, + element.position(), + ); + } + self.emit_at(Op::MakeArray(elements.len() as u16), expr.position()); + } + + // The other half of the same shape. Rhai keeps a map literal as a + // template holding every key — the constant values already in + // place, the computed ones as placeholders — plus the list of + // entries still to evaluate (`ast/expr.rs:283`). An all-constant + // map is folded into a `DynamicConstant` and never arrives here; + // one with a single computed value does, and used to fragment. + Expr::Map(entries, ..) if entries.0.len() <= u16::MAX as usize => { + let (computed, template) = &**entries; + let template = Dynamic::from_map(template.clone()); + // A template whose constants the pool cannot hold is a program + // that could not be written to an artifact anyway. + if !is_poolable(&template) { + self.residual_expr(expr); + return; + } + + self.constant(template); + for (index, (key, value)) in computed.iter().enumerate() { + self.constant(key.name.clone().into()); + self.expression(value); + self.emit_at( + Op::CheckSize { + index: index as u16, + map: true, + }, + value.position(), + ); + } + self.emit_at(Op::MakeMap(computed.len() as u16), expr.position()); + } + + // A block used for its value: `let y = if c { 1 } else { 2 }`, + // `let y = switch ..`, `let y = { let z = 1; z }`. Rhai evaluates + // it with `restore_orig_state` set (`eval/expr.rs:434`), so it + // rewinds what it declared — which is what `block` emits. + Expr::Stmt(block) => { + if !self.block(block.statements()) { + self.defeated = true; + } + } + + // The optimizer folds an all-constant interpolation away before + // this sees it, so what arrives has at least two segments. + Expr::InterpolatedString(segments, ..) => { + self.emit(Op::InterpolateStart); + for segment in segments.iter() { + self.expression(segment); + // The append carries the segment's own position, because + // that is what rhai blames when the size limit goes over. + self.emit_at(Op::InterpolateAppend, segment.position()); + } + self.emit(Op::InterpolateEnd); + } + + // `f.call(x)` and `f.curry(x)` are the method spellings of the two + // above. They arrive as chains, so they have to be taken before + // the chain walker sees them. + // An `rhs` that is a bare `MethodCall` is the whole chain: a + // further step would make it a `Dot` or an `Index` instead. + Expr::Dot(binary, ..) + if matches!(&binary.rhs, Expr::MethodCall(m, ..) + if matches!(m.name.as_str(), "call" | "curry") + && m.args.len() <= u8::MAX as usize) => + { + let Expr::MethodCall(method, ..) = &binary.rhs else { + unreachable!("checked by the guard"); + }; + self.expression(&binary.lhs); + for arg in method.args.iter() { + self.expression(arg); + } + let argc = method.args.len() as u8; + // The call's own position, which is what rhai reports for + // everything the pointer path can raise. The one case it is + // not is `obj.call(x)` where `obj` is not a pointer and `x` is + // taken as one: rhai blames `x` (`func/call.rs:838`). Both + // cannot come from one position-table entry, and using the + // argument's instead was measured to move the divergence onto + // the common path rather than remove it. + // + // Method style only. `curry(f, ..)` written as a call is a + // different path in rhai and takes the *argument's* position — + // see `fn_ptr_call`. The two disagreeing is deliberate. + let pos = binary.rhs.position(); + if method.name == "call" { + self.emit_at(Op::CallFnPtr { argc, method: true }, pos); + } else { + self.emit_at(Op::Curry(argc), pos); + } + } + + Expr::Dot(..) | Expr::Index(..) => { + // A chain emits its own operands, so a failed attempt has to + // leave nothing behind. + let mark = self.mark(); + if !self.chain(expr, Tail::Read, None) { + self.rewind(mark); + self.residual_expr(expr); + } + } + + _ => self.residual_expr(expr), + } + } + + /// Where rhai's method-call rewrite would take this call's first argument + /// from, if it applies at all (`func/call.rs:1434`). + fn receiver(&mut self, call: &FnCallExpr) -> Option { + // An operator short-circuits before the rewrite is reached, and a call + // that captures the enclosing scope is excluded from it outright + // (`func/call.rs:1387` and `:1775`). + if call.op_token.is_some() || call.capture_parent_scope { + return None; + } + + let Some(Expr::Variable(payload, ..)) = call.args.first() else { + return None; + }; + let qualified = has_namespace!(payload); + + match self.slots.resolve(&payload.1) { + Some(slot) if !qualified => Some(Receiver::Local(slot)), + _ if self.is_variable_name(&payload.1, qualified) => { + Some(Receiver::Named(self.push_name(payload.1.clone()))) + } + _ => None, + } + } + + /// Push the arguments left to right, then dispatch. + fn lower_call(&mut self, call: &FnCallExpr, pos: Position) { + let argc = u8::try_from(call.args.len()).expect("checked by is_lowerable_call"); + + // `f(x, ..)` is `x.f(..)`, so the variable is read after the other + // arguments and by reference. See [`Op::CallRef`]. + if let Some(receiver) = self.receiver(call) { + for arg in call.args.iter().skip(1) { + self.expression(arg); + } + // A name is resolved here, where its own position is the one an + // `ErrorVariableNotFound` wants, and then moved under the arguments + // it was read after. + if let Receiver::Named(var) = receiver { + self.emit_at(Op::LoadNamed(var), call.args[0].position()); + if argc > 1 { + self.emit(Op::Rotate(argc - 1)); + } + } + + let name = self.push_name(call.name.clone()); + self.emit_at( + Op::CallRef { + name, + argc, + receiver, + }, + pos, + ); + return; + } + + for arg in call.args.iter() { + self.expression(arg); + } + let name = self.push_name(call.name.clone()); + // Only for a binary operator, which is the only shape the built-in + // lookup takes. Keeping a unary one would be dead weight and worse: + // `UnaryMinus` and `Minus` share the syntax `"-"`, so it is a token + // that cannot be written to an artifact at all. + let op = (argc == 2) + .then(|| call.op_token.clone()) + .flatten() + .map(|token| self.push_token(token)); + self.emit_at(Op::Call { name, argc, op }, pos); + } + + /// Whether a name read is a variable read at all. + /// + /// A qualified name resolves against imported modules rather than the + /// scope, and a bare script-function name is a function pointer with the + /// calling environment attached (`eval/expr.rs:71-99`). Neither is + /// something to look up by name, and both stay fragments. + fn is_variable_name(&self, name: &ImmutableString, qualified: bool) -> bool { + !qualified && !self.script_fns.contains(name) + } + + /// Pool what `x op= y` needs, if there is an operator at all. + fn op_assignment(&mut self, op_info: &OpAssignment) -> Option { + op_info + .get_op_assignment_info() + .map(|(_, _, op_assign, op_assign_str, op, op_str)| { + let entry = AssignOp { + op_assign: op_assign.clone(), + op_assign_name: self.push_name(op_assign_str.into()), + op: op.clone(), + op_name: self.push_name(op_str.into()), + }; + self.push_assign_op(entry) + }) + } + + /// Read a variable without flattening it, leaving a shared cell shared. + /// + /// Rhai's own variable read works this way — `Target::take_or_clone` hands + /// back the shared value untouched (`eval/target.rs:233`) — and the places + /// that want the contents flatten for themselves. [`Op::LoadLocal`] + /// flattens instead, which is right where the value is what matters and + /// wrong in the two places the cell is: + /// + /// * a closure's captured variable, where the aliasing *is* the capture; + /// * a `switch` subject, which rhai refuses to match on when it is not + /// hashable, and a shared value is not — so a shared subject falls to the + /// default arm however well it would otherwise have matched. + fn unflattened(&mut self, expr: &Expr) { + match expr { + Expr::Variable(payload, ..) if !has_namespace!(payload) => { + match self.slots.resolve(&payload.1) { + Some(slot) => self.emit(Op::LoadShared(slot)), + // The caller's. A closure can capture one of those too, and + // reading it flat would bind a copy. + None if self.is_variable_name(&payload.1, false) => { + let name = self.push_name(payload.1.clone()); + self.emit_at(Op::LoadSharedNamed(name), expr.position()); + } + None => self.expression(expr), + } + } + other => self.expression(other), + } + } + + /// Lower `Fn(name)`, `curry(f, ..)` or `call(f, ..)`, if this is one. + /// + /// Rhai resolves these three by name before dispatch, but only at the + /// arities it recognises (`func/call.rs:1109-1245`); anything else is an + /// ordinary call that will not find a function. Matching those arities + /// exactly is what keeps the two agreeing on the failures as well as the + /// successes. + fn fn_ptr_call(&mut self, call: &FnCallExpr, pos: Position) -> bool { + if call_has_namespace!(call) || call.capture_parent_scope { + return false; + } + let argc = call.args.len(); + + match (call.name.as_str(), argc) { + // The argument has to arrive as the cell, not its contents, or the + // answer is always false. + // + // Not lowered under `no_closure`: rhai registers no `is_shared` + // there, so the call has to reach the walker and fail the way rhai + // fails it. Lowering it would answer a question rhai refuses. + #[cfg(not(feature = "no_closure"))] + ("is_shared", 1) => { + self.unflattened(&call.args[0]); + self.emit_at(Op::IsShared, pos); + } + // Both of these are reported against the *argument* rather than + // against the call: rhai reads it, and everything it can then + // complain about — a name that is not a string, a string that is + // not an identifier, a first argument that is not a pointer — is + // filled in with the argument's position (`func/call.rs:1217`, + // `:1220`, `:1232`). + ("Fn", 1) => { + self.expression(&call.args[0]); + self.emit_at(Op::MakeFnPtr, call.args[0].position()); + } + ("curry", _) if argc > 1 => { + let mut args = call.args.iter(); + self.expression(args.next().expect("checked by the arity")); + for arg in args { + // The captured variables. These must bind the *cell* — a + // flattening read would hand the closure a copy and it + // would stop being one. + self.unflattened(arg); + } + self.emit_at(Op::Curry((argc - 1) as u8), call.args[0].position()); + } + ("call", _) if argc >= 1 && argc <= u8::MAX as usize + 1 => { + for arg in call.args.iter() { + self.expression(arg); + } + self.emit_at( + Op::CallFnPtr { + argc: (argc - 1) as u8, + method: false, + }, + pos, + ); + } + _ => return false, + } + true + } + + /// Whether a call can go through generic dispatch. + /// + /// Rhai resolves a handful of names syntactically in `eval_fn_call_expr` + /// before dispatch ever happens (`func/call.rs:1109-1340`), so routing + /// those through `call_fn_raw` would change what they mean. A call that + /// captures the enclosing scope is closure construction, and a qualified + /// name resolves against imported modules; neither is a plain call. + fn is_lowerable_call(&self, call: &FnCallExpr) -> bool { + // `is_shared` belongs here for a sharper reason than the rest: rhai + // answers it syntactically in both call positions (`func/call.rs:1240` + // and `:929`) and registers no function for it anywhere, so a lowered + // call raises `ErrorFunctionNotFound` where the walker returns a bool. + const SYNTACTIC: &[&str] = &["eval", "is_def_var", "is_def_fn"]; + + // These are handled by `fn_ptr_call` above, but only at the arities + // rhai treats syntactically — at any other arity it falls through to + // ordinary dispatch, and so must this. + if matches!(call.name.as_str(), "Fn" | "call" | "curry" | "is_shared") { + return false; + } + + !call.capture_parent_scope + && !call_has_namespace!(call) + && call.args.len() <= u8::MAX as usize + && !SYNTACTIC.contains(&call.name.as_str()) + } + + /// Lower `&&` or `||`: evaluate operands left to right, stopping at the + /// first that decides the result. + /// + /// Each operand is coerced to bool at its own position, which is why the + /// jumps carry one — rhai reports a non-boolean operand against the + /// operand, not the expression (`eval/expr.rs:367-399`). + fn short_circuit(&mut self, operands: &[Expr], stop_on: bool) { + let mut decided = Vec::new(); + + for operand in operands { + self.expression(operand); + let pos = operand.position(); + let site = self.code.len(); + self.emit_at( + if stop_on { + Op::JumpIfTrue { target: u32::MAX } + } else { + Op::JumpIfFalse { target: u32::MAX } + }, + pos, + ); + decided.push(site); + } + + self.emit(Op::Bool(!stop_on)); + let past = self.emit_jump(); + for site in decided { + self.patch_here(site); + } + self.emit(Op::Bool(stop_on)); + self.patch_here(past); + } + + /// Lower a block, leaving its value — the last statement's, or unit if + /// empty — on the stack, and dropping anything it declared. + fn block(&mut self, statements: &[Stmt]) -> bool { + let depth = self.slots.depth(); + + let Some((last, leading)) = statements.split_last() else { + self.emit(Op::Unit); + return true; + }; + + for stmt in leading { + if !self.statement(stmt) { + return false; + } + self.emit(Op::Pop); + } + if !self.statement(last) { + return false; + } + + self.unwind_to(depth); + true + } + + /// Lower a block for its effects only, leaving nothing on the stack. + /// + /// Loop bodies discard their value: rhai's loops yield unit or whatever a + /// `break` supplied, never the body's last statement. + fn block_discarding(&mut self, statements: &[Stmt]) -> bool { + if !self.block(statements) { + return false; + } + self.emit(Op::Pop); + true + } + + /// Emit the scope truncation for leaving a block, and unwind the + /// compile-time slot model with it. + /// + /// The value the block produced is already on the operand stack, so it + /// survives locals being dropped. + fn unwind_to(&mut self, depth: usize) { + if self.slots.depth() > depth { + let depth = u16::try_from(depth).expect("slot count is bounded"); + self.emit(Op::UnwindTo(depth)); + self.slots.unwind_to(depth as usize); + } + } + + /// Where the instruction list currently ends, for [`Lowering::rewind`]. + fn mark(&self) -> usize { + self.code.len() + } + + /// Drop everything emitted since `mark`. + /// + /// Only safe for an attempt that emitted no jumps out of the rewound + /// region, which is why it is used for chains and nothing else: a chain + /// emits its operands and then one instruction, and gives up before + /// emitting the instruction. + fn rewind(&mut self, mark: usize) { + self.code.truncate(mark); + self.positions.truncate(mark); + } + + fn here(&self) -> u32 { + u32::try_from(self.code.len()).expect("chunk length is bounded") + } + + /// Emit a jump with a placeholder target, returning its site for patching. + fn emit_jump(&mut self) -> usize { + let site = self.code.len(); + self.emit(Op::Jump(u32::MAX)); + site + } + + fn emit_jump_if_false(&mut self, pos: Position) -> usize { + let site = self.code.len(); + self.emit_at(Op::JumpIfFalse { target: u32::MAX }, pos); + site + } + + /// Point a previously emitted jump at the next instruction. + fn patch_here(&mut self, site: usize) { + let target = self.here(); + self.patch_to(site, target); + } + + /// Point a previously emitted jump at an instruction already emitted. + fn patch_to(&mut self, site: usize, target: u32) { + match &mut self.code[site] { + Op::Jump(slot) + | Op::JumpIfFalse { target: slot, .. } + | Op::JumpIfTrue { target: slot, .. } + | Op::IterNext { exit: slot, .. } + | Op::PushHandler { target: slot, .. } => *slot = target, + other => unreachable!("patched a {other:?}, which is not a jump"), + } + } + + /// Emit an `IterDrop` for every iterator live above `floor`. + fn drop_iterators(&mut self, floor: usize) { + for _ in floor..self.iters { + self.emit(Op::IterDrop); + } + } + + /// Disarm every `try` region entered above `floor`. + /// + /// A `break` or `continue` jumps over the `PopHandler` the straight-line + /// path would have run. Left armed, the handler keeps a stale target and a + /// stale set of depths, and the next error anywhere in the frame is caught + /// into a `catch` block that has already been left. + fn pop_handlers(&mut self, floor: usize) { + for _ in floor..self.handlers { + self.emit(Op::PopHandler); + } + } + + /// Open a loop whose `break` and `continue` unwind to the same place — + /// `while`, `loop` and `do`, which declare nothing of their own. + fn begin_loop(&mut self, continue_target: u32) { + let depth = u16::try_from(self.slots.depth()).expect("slot count is bounded"); + self.loops.push(Loop { + continue_target, + break_depth: depth, + continue_depth: depth, + iters: self.iters, + handlers: self.handlers, + owns_iterator: false, + breaks: Vec::new(), + }); + } + + /// Open a `for`, which does declare: the loop variable and any counter + /// live between the two depths, so leaving drops them and going round + /// again does not. + fn begin_for(&mut self, continue_target: u32, break_depth: u16) { + self.loops.push(Loop { + continue_target, + break_depth, + continue_depth: u16::try_from(self.slots.depth()).expect("slot count is bounded"), + iters: self.iters, + handlers: self.handlers, + owns_iterator: true, + breaks: Vec::new(), + }); + } + + fn end_loop(&mut self) -> Vec { + self.loops.pop().expect("loop stack is balanced").breaks + } + + fn residual_expr(&mut self, expr: &Expr) { + let residual = self.push_residual(expr.clone()); + self.emit(Op::EvalAst { + residual, + rewind_scope: true, + }); + } + + fn constant(&mut self, value: Dynamic) { + let index = self.push_const(value); + self.emit(Op::Const(index)); + } + + fn push_const(&mut self, value: Dynamic) -> u32 { + // Programs at this scale make a linear scan cheaper than a hash map, + // and it keeps the pool in emission order for readable disassembly. + let rendered = format!("{value:?}"); + if let Some(index) = self + .consts + .iter() + .position(|existing| format!("{existing:?}") == rendered) + { + return index as u32; + } + self.consts.push(value); + (self.consts.len() - 1) as u32 + } + + fn push_name(&mut self, name: ImmutableString) -> u32 { + if let Some(index) = self.names.iter().position(|existing| *existing == name) { + return index as u32; + } + self.names.push(name); + (self.names.len() - 1) as u32 + } + + /// A script uses a handful of distinct operators however many times it + /// mentions them, so the pool stays tiny and a linear scan is right. + fn push_token(&mut self, token: Token) -> u32 { + if let Some(index) = self.tokens.iter().position(|existing| *existing == token) { + return index as u32; + } + self.tokens.push(token); + (self.tokens.len() - 1) as u32 + } + + fn push_assign_op(&mut self, entry: AssignOp) -> u32 { + if let Some(index) = self + .assign_ops + .iter() + .position(|existing| *existing == entry) + { + return index as u32; + } + self.assign_ops.push(entry); + (self.assign_ops.len() - 1) as u32 + } + + fn push_residual(&mut self, expr: Expr) -> u32 { + self.residuals.push(expr); + (self.residuals.len() - 1) as u32 + } + + fn emit(&mut self, op: Op) { + // An upper bound, not the answer: no instruction pushes more than one + // value, so one slot per instruction cannot be too small. The verifier + // replaces it with the measured high water once lowering is done. + self.max_stack = self.max_stack.saturating_add(1); + self.code.push(op); + self.positions.push(Position::NONE); + } + + /// Emit an instruction that can fail against a place in the source. + /// + /// The position goes to the side table rather than into the instruction, so + /// it can be stripped from an artifact without touching the code. + fn emit_at(&mut self, op: Op, pos: Position) { + self.emit(op); + *self.positions.last_mut().expect("just emitted") = pos; + } +} + +/// One step, still as AST. +/// A step, and where rhai would blame it. +/// +/// The position travels with the step rather than being taken from the chain: +/// rhai reports each kind against its own node, and one chain instruction has +/// only one position-table entry between all of them. +enum ChainStep<'a> { + /// The index expression, and the `[` it sits behind — see [`Step::Index`]. + Index(&'a Expr, rhai::Position), + Property( + &'a ( + (ImmutableString, u64), + (ImmutableString, u64), + ImmutableString, + ), + rhai::Position, + ), + Method(&'a FnCallExpr, rhai::Position), +} + +/// Unpick rhai's nested chain encoding into a root and a list of steps. +/// +/// `a.b[i]` is `Dot { lhs: a, rhs: Index { lhs: b, rhs: i } }`: each nested +/// node's `lhs` is the current step's operand and its `rhs` is the +/// continuation, so the list is built by walking `rhs` and taking `lhs` at each +/// level. The innermost `rhs` is the last step rather than a continuation, +/// which is what ends the walk. +/// +/// `ASTFlags::BREAK` is what ends it, and it carries real information: +/// `a[b[0]]` and `a[b][0]` have the same shape, and the flag is the only thing +/// that says the first one's `b[0]` is an index expression rather than two +/// steps (`eval/chaining.rs:698`). +/// +/// Returns `None` for `?.` and `?[]`, which short-circuit on unit rather than +/// stepping, and for a dot onto anything but a property or a method. +fn flatten_chain(expr: &Expr) -> Option<(&Expr, Vec>)> { + /// A chain node's parts: operand side, continuation side, and whether the + /// step it introduces is a property rather than an index. + fn parts(expr: &Expr) -> Option<(&Expr, &Expr, ASTFlags, bool)> { + match expr { + Expr::Dot(binary, flags, ..) => Some((&binary.lhs, &binary.rhs, *flags, true)), + Expr::Index(binary, flags, ..) => Some((&binary.lhs, &binary.rhs, *flags, false)), + _ => None, + } + } + + let (root, mut rest, mut flags, mut dotted) = parts(expr)?; + let mut steps = Vec::new(); + // Rhai's `op_pos`, which is the position of the chain node the step is + // being taken *inside* rather than of the step's operand, and which walks + // down with the recursion (`eval/chaining.rs:695`). + let mut bracket = expr.position(); + + loop { + if flags.intersects(ASTFlags::NEGATED) { + return None; + } + + // `rest` is the continuation only when it is a chain node *and* this + // node is not marked as the last one. Otherwise it is this step's own + // operand — the index expression, or the property being read. + let next = (!flags.intersects(ASTFlags::BREAK)) + .then(|| parts(rest)) + .flatten(); + + let (operand, following) = match next { + Some((operand, _, _, _)) => (operand, Some(rest)), + None => (rest, None), + }; + + steps.push(match (dotted, operand) { + (true, Expr::Property(prop, pos)) => ChainStep::Property(prop, *pos), + (true, Expr::MethodCall(call, pos)) => ChainStep::Method(call, *pos), + // `a.(expr)` is not syntax, so a dot onto anything else is a shape + // the parser only makes for something handled elsewhere. + (true, _) => return None, + (false, index) => ChainStep::Index(index, bracket), + }); + + match following { + Some(node) => { + let (_, next_rest, next_flags, next_dotted) = + parts(node).expect("checked by `next`"); + rest = next_rest; + flags = next_flags; + dotted = next_dotted; + bracket = node.position(); + } + None => break, + } + } + + Some((root, steps)) +} + +/// Whether a function body reaches for `this`. +/// +/// Such a body can still be *called* — rhai keeps its own copy and the call +/// goes through dispatch — it just cannot be a chunk, because a chunk's locals +/// are scope slots and `this` is not in the scope. +fn uses_this(statements: &[Stmt]) -> bool { + let mut found = false; + let path = &mut Vec::new(); + for stmt in statements { + // `walk` hands over the path to the node it is visiting, so the node + // itself is the last entry. + stmt.walk(path, &mut |path| { + if matches!(path.last(), Some(ASTNode::Expr(Expr::ThisPtr(..)))) { + found = true; + } + // Stop descending once the answer is settled. + !found + }); + } + found +} + +/// Wrap statements as a block expression. +/// +/// `Expr::Stmt` is the one shape `eval_expression_tree_raw` routes to +/// `eval_stmt_block` rather than `eval_expr`, which is what lets statements go +/// back through the walker at all. +fn wrap_statements(statements: Vec) -> Expr { + let span = statements.first().zip(statements.last()).map_or_else( + || Span::new(Position::NONE, Position::NONE), + |(first, last)| crate::types::position::Span::new(first.position(), last.position()), + ); + + Expr::Stmt(Box::new(StmtBlock::new_with_span(statements, span))) +} diff --git a/src/grain/compile/poolable.rs b/src/grain/compile/poolable.rs new file mode 100644 index 000000000..05a8a9442 --- /dev/null +++ b/src/grain/compile/poolable.rs @@ -0,0 +1,57 @@ +use core::ops::{Range, RangeInclusive}; + +use rhai::{Array, Blob, Dynamic, Map, INT}; + +/// Whether a constant can live in the artifact's constant pool. +/// +/// Two constraints happen to coincide here, so one check enforces both. +/// +/// The artifact must be loadable in another process, which rules out anything +/// carrying a host `TypeId`, a live `Rc`, or a clock reading: `Variant`, +/// `Shared`, `TimeStamp`. +/// +/// And `FnPtr` is not a value the VM may simply clone into place, even in the +/// same process. Rhai attaches the calling environment when it reads a +/// function pointer out of a constant (`ast/expr.rs:471-482`), so a pointer +/// copied straight from the pool would be missing the module library it was +/// created against. Keeping it out of the pool leaves it as a fragment, which +/// evaluates through the path that does the attaching. +pub(crate) fn is_poolable(value: &Dynamic) -> bool { + // `is_float` follows rhai's own `no_float`; the feature matrix for the + // device build is M6's problem, not this function's. + if value.is_unit() + || value.is_bool() + || value.is_int() + || value.is_char() + || value.is_string() + || value.is_float() + { + return true; + } + + if value.is_array() { + return value + .read_lock::() + .is_some_and(|array| array.iter().all(is_poolable)); + } + + if value.is_map() { + return value + .read_lock::() + .is_some_and(|map| map.values().all(is_poolable)); + } + + if value.is_blob() { + return value.read_lock::().is_some(); + } + + // A range is a host type by representation but not by nature: rhai builds + // one for `0..5` and indexes strings and arrays with it, and its `TypeId` + // is one both sides can name. Without this every slice is a fragment. + if value.is::>() || value.is::>() { + return true; + } + + // Anything else — FnPtr, TimeStamp, Decimal, a host type, a shared cell. + false +} diff --git a/src/grain/compile/slots.rs b/src/grain/compile/slots.rs new file mode 100644 index 000000000..577508a95 --- /dev/null +++ b/src/grain/compile/slots.rs @@ -0,0 +1,48 @@ +use rhai::ImmutableString; + +/// Assigns a slot to every local, mirroring how rhai's own `Scope` grows. +/// +/// Locals live in the caller's `Scope`, and rhai pushes an entry per +/// declaration and truncates back on block exit. So a slot is just the entry's +/// index, and a block boundary is a mark-and-truncate on this side too. +/// +/// Shadowing needs no special handling: a second `let x` pushes a second entry, +/// and resolution scans backwards, so the newer slot wins while the older one +/// stays addressable by anything compiled before it — which is exactly rhai's +/// behaviour. +#[derive(Debug, Default)] +pub(crate) struct Slots { + names: Vec, +} + +impl Slots { + /// Slot count, which is also the scope depth a block should unwind to. + pub(crate) fn depth(&self) -> usize { + self.names.len() + } + + /// Declare a local and return its slot. + pub(crate) fn declare(&mut self, name: ImmutableString) -> u16 { + let slot = self.names.len(); + self.names.push(name); + u16::try_from(slot).expect("slot count is bounded by the compiler's own limit") + } + + /// Resolve a name to the most recent slot holding it. + pub(crate) fn resolve(&self, name: &str) -> Option { + self.names + .iter() + .rposition(|candidate| candidate.as_str() == name) + .map(|slot| u16::try_from(slot).expect("slot count is bounded")) + } + + /// Drop every local declared since `depth`. + pub(crate) fn unwind_to(&mut self, depth: usize) { + self.names.truncate(depth); + } + + /// Whether another local would overflow the slot encoding. + pub(crate) fn is_full(&self) -> bool { + self.names.len() >= u16::MAX as usize + } +} diff --git a/src/grain/format/abi.rs b/src/grain/format/abi.rs new file mode 100644 index 000000000..845d68da3 --- /dev/null +++ b/src/grain/format/abi.rs @@ -0,0 +1,235 @@ +//! What a `Dynamic` is, on the machine that wrote the artifact. +//! +//! Rhai's feature flags change the value representation rather than just what +//! is available: `f32_float` makes `FLOAT` an `f32`, `only_i32` narrows `INT`, +//! `sync` swaps `Rc` for `Arc`. Loading an artifact across one of those is not +//! a missing feature, it is a value decoded as the wrong type — so the header +//! carries a fingerprint and the loader refuses a mismatch by name. +//! +//! ## What the fingerprint can and cannot see +//! +//! Widths are *measured*, so they are right no matter how rhai was configured. +//! The booleans are read from this crate's own features, which is why the +//! manifest mirrors them — `cfg!(feature = "no_object")` here does not consult +//! rhai's manifest. +//! +//! That leaves one gap: enabling a restriction on rhai directly, bypassing the +//! mirror. The cross-checks below close it wherever rust can prove the +//! disagreement, turning it into a compile error rather than a wrong +//! fingerprint. They cannot close it everywhere, which is what the mirror is +//! documented for. + +/// Restrictions that are not visible in a width. +/// +/// Order is the wire order and must never change; append only. A flag's name +/// is what the loader reports, so it has to match rhai's own spelling. +const FLAGS: &[(&str, bool)] = &[ + ("sync", cfg!(feature = "sync")), + ("decimal", cfg!(feature = "decimal")), + ("no_index", cfg!(feature = "no_index")), + ("no_object", cfg!(feature = "no_object")), + ("no_closure", cfg!(feature = "no_closure")), + ("no_function", cfg!(feature = "no_function")), + ("no_module", cfg!(feature = "no_module")), + ("no_position", cfg!(feature = "no_position")), + ("no_custom_syntax", cfg!(feature = "no_custom_syntax")), + ("no_time", cfg!(feature = "no_time")), + ("unchecked", cfg!(feature = "unchecked")), +]; + +/// `Engine` is only `Send + Sync` when rhai is built with `sync`, so claiming +/// the flag without rhai agreeing fails to compile. +#[cfg(feature = "sync")] +const _: () = { + const fn assert_sync() {} + let _ = assert_sync::; +}; + +/// The value representation an artifact was written against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Abi { + /// `size_of::()`. Measured, so `only_i32` set on rhai alone is + /// still caught. + pub int_bytes: u8, + /// `size_of::()`, or 0 under `no_float`. + pub float_bytes: u8, + /// [`FLAGS`] as a bitmask, low bit first. + pub flags: u32, +} + +/// How two fingerprints differ. +/// +/// Naming the difference is the whole point: "artifact was built for a +/// different rhai" is not something a user can act on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AbiMismatch { + /// A width differs, which means integers or floats would decode wrong. + Width { + what: &'static str, + artifact: u8, + host: u8, + }, + /// A restriction differs. `artifact` is whether the writer had it on. + Flag { + flag: &'static str, + artifact: bool, + }, +} + +impl core::fmt::Display for AbiMismatch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Width { + what, + artifact, + host, + } => write!( + f, + "artifact was written with a {artifact}-byte {what}, but this build has {host}" + ), + Self::Flag { flag, artifact } => { + let (writer, reader) = if *artifact { ("on", "off") } else { ("off", "on") }; + write!( + f, + "artifact was written with `{flag}` {writer}, but this build has it {reader}" + ) + } + } + } +} + +impl Abi { + /// The fingerprint of the running build. + #[must_use] + pub fn host() -> Self { + #[cfg(not(feature = "no_float"))] + let float_bytes = core::mem::size_of::() as u8; + #[cfg(feature = "no_float")] + let float_bytes = 0u8; + + let mut flags = 0u32; + for (bit, (_, on)) in FLAGS.iter().enumerate() { + if *on { + flags |= 1 << bit; + } + } + + Self { + int_bytes: core::mem::size_of::() as u8, + float_bytes, + flags, + } + } + + /// Why this fingerprint cannot be loaded by `host`, if it cannot. + /// + /// Widths first: they are measured rather than declared, so they are the + /// claim least likely to be lying. + #[must_use] + pub fn incompatible_with(self, host: Self) -> Option { + if self.int_bytes != host.int_bytes { + return Some(AbiMismatch::Width { + what: "INT", + artifact: self.int_bytes, + host: host.int_bytes, + }); + } + if self.float_bytes != host.float_bytes { + return Some(AbiMismatch::Width { + what: "FLOAT", + artifact: self.float_bytes, + host: host.float_bytes, + }); + } + + let differing = self.flags ^ host.flags; + if differing != 0 { + let bit = differing.trailing_zeros() as usize; + // A bit past the table means the writer knew a flag this build does + // not. Reporting it as unknown beats indexing out of bounds. + let flag = FLAGS.get(bit).map_or("an unknown restriction", |(name, _)| *name); + return Some(AbiMismatch::Flag { + flag, + artifact: self.flags & (1 << bit) != 0, + }); + } + + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_build_can_load_its_own_artifacts() { + assert_eq!(Abi::host().incompatible_with(Abi::host()), None); + } + + #[test] + fn the_build_measures_the_widths_it_should() { + let abi = Abi::host(); + // Against the aliases rather than concrete types, so this holds under + // only_i32 and f32_float as well — the point is that the fingerprint + // reports the build it was taken on, whichever build that is. + assert_eq!( + abi.int_bytes as usize, + core::mem::size_of::(), + ); + #[cfg(not(feature = "no_float"))] + assert_eq!( + abi.float_bytes as usize, + core::mem::size_of::(), + ); + #[cfg(feature = "no_float")] + assert_eq!(abi.float_bytes, 0); + } + + #[test] + fn a_narrower_int_is_refused_by_name() { + let narrow = Abi { + int_bytes: 4, + ..Abi::host() + }; + assert_eq!( + narrow.incompatible_with(Abi::host()), + Some(AbiMismatch::Width { + what: "INT", + artifact: 4, + host: 8, + }), + ); + } + + /// The message has to name the flag; that is the difference between an + /// error a user can act on and one they cannot. + #[test] + fn a_differing_restriction_is_refused_by_name() { + let host = Abi::host(); + let restricted = Abi { + flags: host.flags ^ (1 << 3), + ..host + }; + + let Some(mismatch @ AbiMismatch::Flag { flag, .. }) = restricted.incompatible_with(host) + else { + panic!("a differing flag must be refused"); + }; + assert_eq!(flag, "no_object"); + assert!(mismatch.to_string().contains("no_object")); + } + + #[test] + fn a_flag_this_build_has_never_heard_of_does_not_panic() { + let host = Abi::host(); + let future = Abi { + flags: host.flags ^ (1 << 31), + ..host + }; + assert!(matches!( + future.incompatible_with(host), + Some(AbiMismatch::Flag { .. }), + )); + } +} diff --git a/src/grain/format/mod.rs b/src/grain/format/mod.rs new file mode 100644 index 000000000..97444ddf1 --- /dev/null +++ b/src/grain/format/mod.rs @@ -0,0 +1,315 @@ +//! The on-the-wire form of a [`Program`](crate::Program). +//! +//! This is what the project is for. A device that loads bytes runs no parser +//! and builds no tree, so neither the nodes a retained rhai `AST` costs nor the +//! parser's higher peak is ever spent. Everything else — the speed, the +//! verifier — is downstream of being able to write a program out and read it +//! back somewhere else. +//! +//! ## Shape +//! +//! ```text +//! "RGRN" magic +//! u16 format version +//! abi INT width, FLOAT width, restriction bitmask +//! varint + utf8 source name, empty for none +//! section names +//! section constants +//! section operator tokens +//! section op-assignments +//! section chains +//! section switch tables, prefixed with a hasher probe +//! varint declared max stack +//! section code, verbatim +//! section position table, empty when stripped +//! ``` +//! +//! Everything outside the code section is LEB128, signed values zigzagged, +//! because it is read once at load. The code section is not: it is the bytes +//! the VM executes, copied in and sliced back out untouched, with fixed-width +//! operands so dispatch does not decode. See [`crate::bytecode::code`]. +//! +//! ## What it refuses to write +//! +//! Residual fragments are real `Expr` trees — precisely the allocation this +//! removes — so a program holding any is rejected rather than partially +//! written. A script function the compiler could not lower is rejected for the +//! same reason: rhai keeps its own copy of that one, as an AST. Both failures +//! name what blocked them, because "cannot serialize" without the construct is +//! not something a script author can act on. + +use core::convert::TryFrom; + +mod abi; +mod read; +mod write; + +pub use abi::{Abi, AbiMismatch}; +pub use read::ReadError; +pub use write::WriteError; + +use crate::grain::bytecode::VerifyError; +use crate::grain::program::Program; + +/// Identifies the format, so a file that is not one fails immediately rather +/// than as a nonsense opcode. +const MAGIC: [u8; 4] = *b"RGRN"; + +/// Bumped when an encoding changes in a way an older reader would misread. +/// Additive changes that an older reader would reject anyway — a new op tag, +/// a new constant tag — do not need it. +const VERSION: u16 = 6; + +/// Where a chain starts. Append only. +mod root_tag { + pub const LOCAL: u8 = 0x01; + pub const TEMPORARY: u8 = 0x02; + pub const NAMED: u8 = 0x03; +} + +/// Chain-step tags. Append only. +mod step_tag { + pub const INDEX: u8 = 0x01; + pub const PROPERTY: u8 = 0x02; + pub const METHOD: u8 = 0x03; +} + +/// What a chain does at the end. Append only. +mod tail_tag { + pub const READ: u8 = 0x01; + pub const ASSIGN: u8 = 0x02; + pub const ASSIGN_OP: u8 = 0x03; +} + +/// Constant-pool tags, over the subset of `Dynamic` that means the same thing +/// in another process. Append only. +mod constant { + pub const UNIT: u8 = 0x00; + pub const FALSE: u8 = 0x01; + pub const TRUE: u8 = 0x02; + pub const INT: u8 = 0x03; + pub const FLOAT: u8 = 0x04; + pub const CHAR: u8 = 0x05; + pub const STRING: u8 = 0x06; + pub const ARRAY: u8 = 0x07; + pub const MAP: u8 = 0x08; + pub const BLOB: u8 = 0x09; + pub const RANGE: u8 = 0x0a; + pub const RANGE_INCLUSIVE: u8 = 0x0b; +} + +impl<'a> Program<'a> { + /// Encode this program, diagnostics included. + /// + /// # Errors + /// + /// Fails if the program still holds anything that cannot cross a process + /// boundary: an un-lowered fragment, a script function, or a constant + /// carrying a host type. + pub fn write(&self) -> Result, WriteError> { + write::write(self, write::Positions::Keep) + } + + /// Encode this program without its position table, returning the table + /// separately. + /// + /// This is the split the debug layer exists for. Ship the first half to the + /// device and keep the second: errors then arrive carrying an instruction + /// address, and [`rhaigrain_pos::resolve`] turns it back into a position + /// where the source still is. The table can also be sent back later with + /// [`Program::attach_positions`]. + /// + /// # Errors + /// + /// As [`Program::write`]. + pub fn write_stripped(&self) -> Result<(Vec, Vec), WriteError> { + let bytes = write::write(self, write::Positions::Strip)?; + Ok((bytes, self.positions().to_table())) + } + + /// Decode a program written by [`Program::write`], borrowing its + /// instructions from `bytes`. + /// + /// Nothing is allocated for the code — the returned program points into the + /// buffer and the VM dispatches on it where it lies. What is allocated is + /// bounded by the distinct constants, names and operators the script + /// mentions, not by how long it is. Call [`Program::into_owned`] if the + /// buffer has to go. + /// + /// The chunk is verified before this returns, so a program that loads + /// cannot underflow the operand stack, jump outside itself, jump into the + /// middle of an instruction, or index a pool entry that is not there. With + /// the code being executed in place, that check is the only thing between a + /// corrupt file and an operand read as an opcode. + /// + /// # Errors + /// + /// Fails on a bad header, an ABI the running build cannot represent, + /// truncated or malformed input, or a chunk that does not verify. + pub fn read(bytes: &'a [u8]) -> Result { + read::read(bytes) + } +} + +/// A reader positioned in a byte slice. +struct Cursor<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, pos: 0 } + } + + fn take(&mut self, n: usize) -> Result<&'a [u8], ReadError> { + let end = self.pos.checked_add(n).ok_or(ReadError::Truncated)?; + let slice = self.bytes.get(self.pos..end).ok_or(ReadError::Truncated)?; + self.pos = end; + Ok(slice) + } + + fn byte(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + /// A count that has to be reserved for before it is read. + /// + /// Nothing is encoded in less than a byte, so a count larger than what is + /// left cannot be honest — and reserving for it first would let a handful + /// of bytes ask for a terabyte. Reading the entries one at a time runs out + /// of input safely; `Vec::with_capacity` does not, because it allocates + /// before the first entry is read. Found by `tests/fuzz.rs`. + fn count(&mut self) -> Result { + let count = usize::try_from(self.uvarint()?).map_err(|_| ReadError::Truncated)?; + if count > self.bytes.len() - self.pos { + return Err(ReadError::Truncated); + } + Ok(count) + } + + /// LEB128, capped at ten groups so a run of continuation bytes cannot spin. + fn uvarint(&mut self) -> Result { + let mut value = 0u64; + for shift in (0..64).step_by(7) { + let byte = self.byte()?; + value |= u64::from(byte & 0x7f) + .checked_shl(shift) + .ok_or(ReadError::MalformedVarint)?; + if byte & 0x80 == 0 { + return Ok(value); + } + } + Err(ReadError::MalformedVarint) + } + + fn ivarint(&mut self) -> Result { + let raw = self.uvarint()?; + Ok(((raw >> 1) as i64) ^ -((raw & 1) as i64)) + } + + fn index(&mut self) -> Result { + u32::try_from(self.uvarint()?).map_err(|_| ReadError::MalformedVarint) + } + + fn small(&mut self) -> Result { + u16::try_from(self.uvarint()?).map_err(|_| ReadError::MalformedVarint) + } + + fn str(&mut self) -> Result<&'a str, ReadError> { + let len = usize::try_from(self.uvarint()?).map_err(|_| ReadError::Truncated)?; + core::str::from_utf8(self.take(len)?).map_err(|_| ReadError::BadUtf8) + } + + fn at_end(&self) -> bool { + self.pos == self.bytes.len() + } +} + +fn put_uvarint(out: &mut Vec, mut value: u64) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +fn put_ivarint(out: &mut Vec, value: i64) { + put_uvarint(out, ((value << 1) ^ (value >> 63)) as u64); +} + +fn put_str(out: &mut Vec, value: &str) { + put_uvarint(out, value.len() as u64); + out.extend_from_slice(value.as_bytes()); +} + +impl From for ReadError { + fn from(err: VerifyError) -> Self { + Self::Unverifiable(err) + } +} + +impl From for ReadError { + fn from(err: crate::grain::bytecode::BadTable) -> Self { + Self::Names(err) + } +} + +impl From for ReadError { + fn from(err: crate::grain::bytecode::TableError) -> Self { + Self::Positions(err) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Varints are the whole encoding's foundation; a rounding error here + /// misreads every index in the file. + #[test] + fn unsigned_varints_round_trip_at_the_edges() { + for value in [0u64, 1, 127, 128, 300, u32::MAX as u64, u64::MAX] { + let mut buf = Vec::new(); + put_uvarint(&mut buf, value); + assert_eq!(Cursor::new(&buf).uvarint().unwrap(), value, "at {value}"); + } + } + + #[test] + fn signed_varints_round_trip_across_zero() { + for value in [0i64, -1, 1, -64, 63, i32::MIN as i64, i64::MIN, i64::MAX] { + let mut buf = Vec::new(); + put_ivarint(&mut buf, value); + assert_eq!(Cursor::new(&buf).ivarint().unwrap(), value, "at {value}"); + } + } + + /// Small numbers are most of a chunk, so the encoding only pays for itself + /// if they cost one byte. + #[test] + fn small_indices_cost_one_byte() { + let mut buf = Vec::new(); + put_uvarint(&mut buf, 127); + assert_eq!(buf.len(), 1); + } + + #[test] + fn a_run_of_continuation_bytes_terminates() { + let never_ends = vec![0xffu8; 64]; + assert_eq!( + Cursor::new(&never_ends).uvarint(), + Err(ReadError::MalformedVarint), + ); + } + + #[test] + fn reading_past_the_end_is_an_error_not_a_panic() { + assert_eq!(Cursor::new(&[]).byte(), Err(ReadError::Truncated)); + assert_eq!(Cursor::new(&[1, 2]).take(9), Err(ReadError::Truncated)); + } +} diff --git a/src/grain/format/read.rs b/src/grain/format/read.rs new file mode 100644 index 000000000..5e5838935 --- /dev/null +++ b/src/grain/format/read.rs @@ -0,0 +1,453 @@ +use crate::{tokenizer::Token, Dynamic, ImmutableString}; +use core::convert::{TryFrom, TryInto}; + +use crate::grain::bytecode::{ + AssignOp, BadTable, Chain, Chunk, Positions, Root, Step, Strings, Switch, SwitchCase, + SwitchRange, TableError, Tail, VerifyError, +}; +use crate::grain::format::abi::{Abi, AbiMismatch}; +use crate::grain::format::{constant, root_tag, step_tag, tail_tag, Cursor, MAGIC, VERSION}; +use crate::grain::program::{Function, Parts, Program}; + +/// How deeply a constant may nest. +/// +/// Decoding an array or a map recurses, so an artifact claiming a few thousand +/// nested arrays would overflow the stack of whatever loads it. Rhai's own +/// parser caps expression depth for the same reason; this is the loader's +/// version, and it is well past anything a literal in real source reaches. +const MAX_CONSTANT_DEPTH: usize = 64; + +/// Why an artifact could not be loaded. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadError { + /// Not a rhaigrain artifact at all. + BadMagic, + /// Written by a format this build does not know how to read. + UnsupportedVersion { found: u16, supported: u16 }, + /// Written against a different value representation. Loading anyway would + /// decode integers or floats as the wrong type. + Abi(AbiMismatch), + /// The input ended mid-value. + Truncated, + /// A varint that never terminates, or one too wide for its field. + MalformedVarint, + /// A string that is not UTF-8. + BadUtf8, + /// A tag this build has no meaning for. + UnknownTag { section: &'static str, tag: u8 }, + /// An operator syntax rhai does not recognise. + UnknownToken { syntax: String }, + /// The artifact's `switch` case hashes were computed by a differently + /// seeded hasher, so none of them would ever match. + HashSeedMismatch { artifact: u64, host: u64 }, + /// Constants nested past [`MAX_CONSTANT_DEPTH`]. + ConstantTooDeep, + /// Bytes left over after the last section, so the file is not what it + /// claims to be even though every field parsed. + TrailingBytes { count: usize }, + /// The chunk parsed but does not agree with itself. + Unverifiable(VerifyError), + /// The position table is malformed, or belongs to a different program. + Positions(TableError), + /// The name table's spans do not fit its blob. + Names(BadTable), +} + +impl core::fmt::Display for ReadError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::BadMagic => f.write_str("not a rhaigrain artifact"), + Self::UnsupportedVersion { found, supported } => write!( + f, + "artifact is format version {found}, and this build reads {supported}" + ), + Self::Abi(mismatch) => write!(f, "{mismatch}"), + Self::Truncated => f.write_str("artifact ends mid-value"), + Self::MalformedVarint => f.write_str("malformed varint"), + Self::BadUtf8 => f.write_str("a string is not valid UTF-8"), + Self::UnknownTag { section, tag } => { + write!(f, "unknown {section} tag {tag:#04x}") + } + Self::UnknownToken { syntax } => write!(f, "`{syntax}` is not an operator"), + Self::HashSeedMismatch { artifact, host } => write!( + f, + "this artifact's `switch` cases were hashed with a different seed \ + ({artifact:#018x} against {host:#018x}), so none of them could match — \ + call `rhai::config::hashing::set_hashing_seed` with the same seed \ + wherever this was compiled and wherever it is loaded" + ), + Self::ConstantTooDeep => write!( + f, + "a constant nests deeper than {MAX_CONSTANT_DEPTH} levels" + ), + Self::TrailingBytes { count } => { + write!(f, "{count} byte(s) follow the last section") + } + Self::Unverifiable(err) => write!(f, "chunk failed verification: {err:?}"), + Self::Positions(err) => write!(f, "{err}"), + Self::Names(err) => write!(f, "name table is malformed: {err:?}"), + } + } +} + +pub(super) fn read(bytes: &[u8]) -> Result, ReadError> { + let mut cursor = Cursor::new(bytes); + + if cursor.take(MAGIC.len())? != MAGIC { + return Err(ReadError::BadMagic); + } + + let version = u16::from_le_bytes(cursor.take(2)?.try_into().expect("two bytes")); + if version != VERSION { + return Err(ReadError::UnsupportedVersion { + found: version, + supported: VERSION, + }); + } + + // Before anything is decoded: past here every value is read as a type the + // fingerprint just promised. + let abi = Abi { + int_bytes: cursor.byte()?, + float_bytes: cursor.byte()?, + flags: u32::from_le_bytes(cursor.take(4)?.try_into().expect("four bytes")), + }; + if let Some(mismatch) = abi.incompatible_with(Abi::host()) { + return Err(ReadError::Abi(mismatch)); + } + + let source = cursor.str()?; + let source = (!source.is_empty()).then(|| ImmutableString::from(source)); + + // Borrowed: the spans are read, the blob is sliced, and nothing per-name + // is allocated. + let count = cursor.count()?; + let mut starts = Vec::with_capacity(count + 1); + starts.push(0u32); + for _ in 0..count { + starts.push(cursor.index()?); + } + let blob_len = usize::try_from(cursor.uvarint()?).map_err(|_| ReadError::Truncated)?; + let names = Strings::borrowed(cursor.take(blob_len)?, starts)?; + + let mut consts = Vec::new(); + for _ in 0..cursor.uvarint()? { + consts.push(get_constant(&mut cursor, 0)?); + } + + let mut tokens = Vec::new(); + for _ in 0..cursor.uvarint()? { + tokens.push(get_token(&mut cursor)?); + } + + let mut assign_ops = Vec::new(); + for _ in 0..cursor.uvarint()? { + assign_ops.push(AssignOp { + op_assign: get_token(&mut cursor)?, + op_assign_name: cursor.index()?, + op: get_token(&mut cursor)?, + op_name: cursor.index()?, + }); + } + + let mut chains = Vec::new(); + for _ in 0..cursor.uvarint()? { + chains.push(get_chain(&mut cursor)?); + } + + let switches = get_switches(&mut cursor)?; + + let main = get_chunk(&mut cursor)?; + + let mut functions = Vec::new(); + for _ in 0..cursor.uvarint()? { + let name = cursor.index()?; + let mut params = Vec::new(); + for _ in 0..cursor.uvarint()? { + params.push(cursor.index()?); + } + functions.push(Function { + name, + params, + chunk: get_chunk(&mut cursor)?, + }); + } + + // Borrowed, not copied. The VM dispatches on these bytes where they lie. + let code_len = usize::try_from(cursor.uvarint()?).map_err(|_| ReadError::Truncated)?; + let code = cursor.take(code_len)?; + + // Absent means stripped, which is the normal shape for something that + // reached a device. `from_table` refuses a table belonging to another + // program, so a mismatched pair fails here rather than misreporting later. + let table_len = usize::try_from(cursor.uvarint()?).map_err(|_| ReadError::Truncated)?; + let positions = if table_len == 0 { + Positions::Stripped + } else { + Positions::from_table(cursor.take(table_len)?, code_len)? + }; + + if !cursor.at_end() { + return Err(ReadError::TrailingBytes { + count: bytes.len() - cursor.pos, + }); + } + + let program = Program::new( + code.into(), + main, + functions, + Parts { + positions, + residuals: Vec::new(), + consts, + names, + tokens, + assign_ops, + chains, + switches, + // Script functions are still ASTs, so `write` refuses a program + // that has any and a loaded one never does. + lib: None, + #[cfg(not(feature = "no_module"))] + resolver: None, + source, + }, + ); + + // An artifact is untrusted input, so the chunk is checked before it is + // handed to a `Vm`. Nothing that fails here is constructible by the + // compiler — and because the VM executes these bytes in place, this is what + // stands between a corrupt file and an operand read as an opcode. + program.verify()?; + + Ok(program) +} + +/// A chain step's own position. Line zero means none. +fn get_position(cursor: &mut Cursor) -> Result { + let line = cursor.small()?; + let column = cursor.small()?; + Ok(if line == 0 { + rhai::Position::NONE + } else { + rhai::Position::new(line, column) + }) +} + +fn get_chain(cursor: &mut Cursor) -> Result { + let root = match cursor.byte()? { + root_tag::LOCAL => Root::Local { + slot: cursor.small()?, + name: cursor.index()?, + }, + root_tag::NAMED => Root::Named { + name: cursor.index()?, + pos: get_position(cursor)?, + }, + root_tag::TEMPORARY => Root::Temporary, + tag => { + return Err(ReadError::UnknownTag { + section: "chain root", + tag, + }) + } + }; + let operands = cursor.small()?; + + let mut steps = Vec::new(); + for _ in 0..cursor.uvarint()? { + steps.push(match cursor.byte()? { + step_tag::INDEX => Step::Index { + operand: cursor.small()?, + pos: get_position(cursor)?, + bracket: get_position(cursor)?, + }, + step_tag::PROPERTY => Step::Property { + name: cursor.index()?, + getter: cursor.index()?, + setter: cursor.index()?, + pos: get_position(cursor)?, + }, + step_tag::METHOD => Step::Method { + name: cursor.index()?, + argc: cursor.byte()?, + operand: cursor.small()?, + pos: get_position(cursor)?, + }, + tag => { + return Err(ReadError::UnknownTag { + section: "chain step", + tag, + }) + } + }); + } + + let tail = match cursor.byte()? { + tail_tag::READ => Tail::Read, + tail_tag::ASSIGN => Tail::Assign { op: None }, + tail_tag::ASSIGN_OP => Tail::Assign { + op: Some(cursor.index()?), + }, + tag => { + return Err(ReadError::UnknownTag { + section: "chain tail", + tag, + }) + } + }; + + Ok(Chain { + root, + steps, + tail, + operands, + }) +} + +/// Read the switch tables, refusing them if their case hashes were made by a +/// hasher this process cannot reproduce. +/// +/// The check is not belt and braces: without it a seed mismatch loads cleanly +/// and every `switch` silently takes its default, which is a wrong answer +/// rather than a failure. See `write::put_switches`. +fn get_switches(cursor: &mut Cursor) -> Result, ReadError> { + let count = cursor.uvarint()?; + if count == 0 { + return Ok(Vec::new()); + } + + let artifact = u64::from_le_bytes(cursor.take(8)?.try_into().expect("eight bytes")); + let host = crate::grain::bytecode::probe(); + if artifact != host { + return Err(ReadError::HashSeedMismatch { artifact, host }); + } + + let mut switches = Vec::new(); + for _ in 0..count { + let mut cases = Vec::new(); + for _ in 0..cursor.uvarint()? { + cases.push(SwitchCase { + hash: u64::from_le_bytes(cursor.take(8)?.try_into().expect("eight bytes")), + target: cursor.index()?, + }); + } + + let mut ranges = Vec::new(); + for _ in 0..cursor.uvarint()? { + ranges.push(SwitchRange { + from: bounded_int(cursor.ivarint()?)?, + to: bounded_int(cursor.ivarint()?)?, + inclusive: cursor.byte()? != 0, + target: cursor.index()?, + }); + } + + switches.push(Switch { + cases, + ranges, + default: cursor.index()?, + }); + } + Ok(switches) +} + +/// Narrow a written bound back to this build's `INT`. +/// +/// The ABI fingerprint has already promised the widths agree, so this can only +/// fail on a corrupt file — but a range bound is compared against a subject, +/// and a silently truncated one would match the wrong values. +fn bounded_int(value: i64) -> Result { + rhai::INT::try_from(value).map_err(|_| ReadError::MalformedVarint) +} + +/// Read a chunk's span. The verifier is what checks it names real code. +fn get_chunk(cursor: &mut Cursor) -> Result { + let entry = cursor.index()?; + let end = cursor.index()?; + Ok(Chunk::new(entry, end, cursor.small()?)) +} + +fn get_token(cursor: &mut Cursor) -> Result { + let syntax = cursor.str()?; + Token::lookup_symbol_from_syntax(syntax).ok_or_else(|| ReadError::UnknownToken { + syntax: syntax.to_string(), + }) +} + +fn get_constant(cursor: &mut Cursor, depth: usize) -> Result { + if depth > MAX_CONSTANT_DEPTH { + return Err(ReadError::ConstantTooDeep); + } + + Ok(match cursor.byte()? { + constant::UNIT => Dynamic::UNIT, + constant::FALSE => Dynamic::from(false), + constant::TRUE => Dynamic::from(true), + + constant::INT => { + let value = cursor.ivarint()?; + Dynamic::from(rhai::INT::try_from(value).map_err(|_| ReadError::MalformedVarint)?) + } + + #[cfg(not(feature = "no_float"))] + constant::FLOAT => { + let width = core::mem::size_of::(); + let bits = cursor.take(width)?; + Dynamic::from(rhai::FLOAT::from_le_bytes( + bits.try_into().expect("width matches the fingerprint"), + )) + } + + constant::CHAR => { + let code = cursor.index()?; + Dynamic::from(char::from_u32(code).ok_or(ReadError::MalformedVarint)?) + } + + constant::STRING => Dynamic::from(ImmutableString::from(cursor.str()?)), + + constant::ARRAY => { + // The declared length is untrusted, so nothing is reserved from it; + // a short file runs out of bytes instead of out of memory. + let count = cursor.uvarint()?; + let mut array = rhai::Array::new(); + for _ in 0..count { + array.push(get_constant(cursor, depth + 1)?); + } + Dynamic::from(array) + } + + constant::MAP => { + let count = cursor.uvarint()?; + let mut map = rhai::Map::new(); + for _ in 0..count { + let key = cursor.str()?.into(); + map.insert(key, get_constant(cursor, depth + 1)?); + } + Dynamic::from(map) + } + + constant::RANGE => { + let start = bounded_int(cursor.ivarint()?)?; + Dynamic::from(start..bounded_int(cursor.ivarint()?)?) + } + + constant::RANGE_INCLUSIVE => { + let start = bounded_int(cursor.ivarint()?)?; + Dynamic::from(start..=bounded_int(cursor.ivarint()?)?) + } + + constant::BLOB => { + let len = usize::try_from(cursor.uvarint()?).map_err(|_| ReadError::Truncated)?; + Dynamic::from(cursor.take(len)?.to_vec()) + } + + tag => { + return Err(ReadError::UnknownTag { + section: "constant", + tag, + }) + } + }) +} diff --git a/src/grain/format/write.rs b/src/grain/format/write.rs new file mode 100644 index 000000000..c8b5aa453 --- /dev/null +++ b/src/grain/format/write.rs @@ -0,0 +1,426 @@ +use core::ops::{Range, RangeInclusive}; + +use rhai::{tokenizer::Token, Array, Blob, Dynamic, Map, INT}; + +use crate::grain::bytecode::{AssignOp, Chain, Root, Step, Tail}; +use crate::grain::format::abi::Abi; +use crate::grain::format::{ + constant, put_ivarint, put_str, put_uvarint, root_tag, step_tag, tail_tag, MAGIC, VERSION, +}; +use crate::grain::program::Program; + +/// Why a program cannot be written out. +/// +/// Every variant names the construct that blocked it. A serializer that only +/// says "no" leaves the author guessing which line to change. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteError { + /// The program still hands fragments to rhai's walker, and a fragment is a + /// real `Expr` tree. + /// + /// Names the first construct responsible and where it is, because a caller + /// deciding whether to ship source instead needs to know what it is + /// falling back for. + HasResiduals { + count: usize, + construct: &'static str, + pos: rhai::Position, + }, + /// Script functions are still ASTs in the program's library. Compiling them + /// to chunks is M4. + HasScriptFunctions, + /// A pooled constant carries something that has no meaning in another + /// process — a host type, a function pointer, a clock reading. + UnserializableConstant { index: usize, type_name: String }, + /// An operator token that does not survive `syntax -> token`. Storing it + /// would silently change which built-in the VM reaches. + AmbiguousToken { token: String }, +} + +impl core::fmt::Display for WriteError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::HasResiduals { + count, + construct, + pos, + } => write!( + f, + "{construct} at {pos} is not compiled yet, so this program still has \ + {count} fragment(s) that only rhai's walker can evaluate" + ), + Self::HasScriptFunctions => { + f.write_str("script functions are still ASTs and cannot be written") + } + Self::UnserializableConstant { index, type_name } => write!( + f, + "constant {index} is a `{type_name}`, which has no meaning in another process" + ), + Self::AmbiguousToken { token } => { + write!(f, "operator token `{token}` does not survive a round trip") + } + } + } +} + +/// Whether an artifact carries its own diagnostics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Positions { + Keep, + Strip, +} + +pub(super) fn write(program: &Program, positions: Positions) -> Result, WriteError> { + // Refuse before encoding anything, so a rejection cannot leave a caller + // holding a half-written buffer that happens to parse. + if program.residual_count() > 0 { + let (construct, pos) = program + .first_unsupported() + .unwrap_or(("an unlowered expression", rhai::Position::NONE)); + return Err(WriteError::HasResiduals { + count: program.residual_count(), + construct, + pos, + }); + } + if program.lib().is_some() { + return Err(WriteError::HasScriptFunctions); + } + + let mut out = Vec::new(); + out.extend_from_slice(&MAGIC); + out.extend_from_slice(&VERSION.to_le_bytes()); + + let abi = Abi::host(); + out.push(abi.int_bytes); + out.push(abi.float_bytes); + out.extend_from_slice(&abi.flags.to_le_bytes()); + + put_str(&mut out, program.source().map_or("", |s| s.as_str())); + + // One blob and a list of spans, so a loader can point at the names where + // they lie instead of allocating one box per name. + let names = program.names(); + put_uvarint(&mut out, names.len() as u64); + for start in names.starts().iter().skip(1) { + put_uvarint(&mut out, u64::from(*start)); + } + put_uvarint(&mut out, names.blob().len() as u64); + out.extend_from_slice(names.blob()); + + put_uvarint(&mut out, program.consts().len() as u64); + for (index, value) in program.consts().iter().enumerate() { + put_constant(&mut out, value) + .map_err(|type_name| WriteError::UnserializableConstant { index, type_name })?; + } + + put_uvarint(&mut out, program.tokens().len() as u64); + for token in program.tokens() { + put_token(&mut out, token)?; + } + + put_uvarint(&mut out, program.assign_ops().len() as u64); + for entry in program.assign_ops() { + put_assign_op(&mut out, entry)?; + } + + put_uvarint(&mut out, program.chains().len() as u64); + for chain in program.chains() { + put_chain_spec(&mut out, chain); + } + + put_switches(&mut out, program.switches()); + + // Chunks: main first, then one per compiled function. Entry offsets are + // into the single code buffer below. + put_chunk(&mut out, program.main()); + put_uvarint(&mut out, program.functions().len() as u64); + for function in program.functions() { + put_uvarint(&mut out, u64::from(function.name)); + put_uvarint(&mut out, function.params.len() as u64); + for param in &function.params { + put_uvarint(&mut out, u64::from(*param)); + } + put_chunk(&mut out, &function.chunk); + } + + // Verbatim. This is the whole point of the byte encoding: what a loader + // hands the VM is a slice of the artifact, not something rebuilt from it. + let code = program.code(); + put_uvarint(&mut out, code.len() as u64); + out.extend_from_slice(code); + + // Last, and length-prefixed, so removing it is a truncation rather than a + // re-encode — and so a reader that finds nothing there is reading a + // deliberately stripped artifact, not a damaged one. + let table = match positions { + Positions::Keep => program.positions().to_table(), + Positions::Strip => Vec::new(), + }; + put_uvarint(&mut out, table.len() as u64); + out.extend_from_slice(&table); + + Ok(out) +} + +/// A step's position, which travels with the step rather than in the position +/// table — see [`Step::pos`]. Line zero means none, because rhai's own line +/// numbers start at one. +fn put_position(out: &mut Vec, pos: rhai::Position) { + put_uvarint(out, pos.line().unwrap_or(0) as u64); + put_uvarint(out, pos.position().unwrap_or(0) as u64); +} + +fn put_chain_spec(out: &mut Vec, chain: &Chain) { + match chain.root { + Root::Local { slot, name } => { + out.push(root_tag::LOCAL); + put_uvarint(out, u64::from(slot)); + put_uvarint(out, u64::from(name)); + } + Root::Named { name, pos } => { + out.push(root_tag::NAMED); + put_uvarint(out, u64::from(name)); + put_position(out, pos); + } + Root::Temporary => out.push(root_tag::TEMPORARY), + } + put_uvarint(out, u64::from(chain.operands)); + + put_uvarint(out, chain.steps.len() as u64); + for step in &chain.steps { + match step { + Step::Index { + operand, + pos, + bracket, + } => { + out.push(step_tag::INDEX); + put_uvarint(out, u64::from(*operand)); + put_position(out, *pos); + put_position(out, *bracket); + } + Step::Property { + name, + getter, + setter, + pos, + } => { + out.push(step_tag::PROPERTY); + put_uvarint(out, u64::from(*name)); + put_uvarint(out, u64::from(*getter)); + put_uvarint(out, u64::from(*setter)); + put_position(out, *pos); + } + Step::Method { + name, + argc, + operand, + pos, + } => { + out.push(step_tag::METHOD); + put_uvarint(out, u64::from(*name)); + out.push(*argc); + put_uvarint(out, u64::from(*operand)); + put_position(out, *pos); + } + } + } + + match &chain.tail { + Tail::Read => out.push(tail_tag::READ), + Tail::Assign { op: None } => out.push(tail_tag::ASSIGN), + Tail::Assign { op: Some(op) } => { + out.push(tail_tag::ASSIGN_OP); + put_uvarint(out, u64::from(*op)); + } + } +} + +/// Write the switch tables, behind a probe that says whether their hashes will +/// still mean anything on the other side. +/// +/// Rhai's parser keeps only the *hash* of a case value (`ast/stmt.rs:336`), so +/// there is nothing here to re-hash at load — and by default those hashes do +/// not survive the trip, because rhai's default features include +/// `ahash/runtime-rng` and the seed is drawn per process. An artifact with a +/// `switch` in it therefore requires `config::hashing::set_hashing_seed` with +/// the same seed on both sides. +/// +/// The probe is what turns that from a silent wrong answer — every subject +/// dispatched to the default — into a refusal to load. It goes here rather +/// than in the ABI fingerprint because it only constrains artifacts that +/// actually contain a `switch`; making every program agree about a hashing +/// seed would be a restriction bought for nothing. +fn put_switches(out: &mut Vec, switches: &[crate::grain::bytecode::Switch]) { + put_uvarint(out, switches.len() as u64); + if switches.is_empty() { + return; + } + out.extend_from_slice(&crate::grain::bytecode::probe().to_le_bytes()); + + for switch in switches { + put_uvarint(out, switch.cases.len() as u64); + for case in &switch.cases { + // Fixed width: a hash is eight bytes of noise, which a varint + // would spend ten on. + out.extend_from_slice(&case.hash.to_le_bytes()); + put_uvarint(out, u64::from(case.target)); + } + + put_uvarint(out, switch.ranges.len() as u64); + for range in &switch.ranges { + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(range.from)); + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(range.to)); + out.push(u8::from(range.inclusive)); + put_uvarint(out, u64::from(range.target)); + } + + put_uvarint(out, u64::from(switch.default)); + } +} + +fn put_chunk(out: &mut Vec, chunk: &crate::grain::bytecode::Chunk) { + put_uvarint(out, u64::from(chunk.entry())); + put_uvarint(out, u64::from(chunk.end())); + put_uvarint(out, u64::from(chunk.max_stack())); +} + +/// Store an operator token as its syntax, and prove that reading it back gives +/// the same token. +/// +/// The check is not ceremony. `Plus` and `UnaryPlus` share the syntax `"+"`, +/// so the reverse lookup collapses them — and the token is what the built-in +/// operator lookup keys on, so a collapsed one would quietly reach a different +/// implementation. +fn put_token(out: &mut Vec, token: &Token) -> Result<(), WriteError> { + let ambiguous = || WriteError::AmbiguousToken { + token: format!("{token:?}"), + }; + + if !token.is_literal() { + return Err(ambiguous()); + } + let syntax = token.literal_syntax(); + if Token::lookup_symbol_from_syntax(syntax).as_ref() != Some(token) { + return Err(WriteError::AmbiguousToken { + token: syntax.to_string(), + }); + } + put_str(out, syntax); + Ok(()) +} + +fn put_assign_op(out: &mut Vec, entry: &AssignOp) -> Result<(), WriteError> { + put_token(out, &entry.op_assign)?; + put_uvarint(out, u64::from(entry.op_assign_name)); + put_token(out, &entry.op)?; + put_uvarint(out, u64::from(entry.op_name)); + Ok(()) +} + +/// Encode a constant, or name the type that stopped it. +/// +/// The accepted set is `compile::poolable::is_poolable`'s, which the compiler +/// already applies when filling the pool — so a rejection here means the two +/// have drifted apart, not that a script did something exotic. +fn put_constant(out: &mut Vec, value: &Dynamic) -> Result<(), String> { + if value.is_unit() { + out.push(constant::UNIT); + return Ok(()); + } + if let Ok(flag) = value.as_bool() { + out.push(if flag { + constant::TRUE + } else { + constant::FALSE + }); + return Ok(()); + } + if let Ok(number) = value.as_int() { + out.push(constant::INT); + // `INT` narrows to `i32` under `only_i32`, so the widening is real + // there even though it is a no-op here. The fingerprint is what stops + // the reader from decoding the wrong width back. + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(number)); + return Ok(()); + } + #[cfg(not(feature = "no_float"))] + if let Ok(number) = value.as_float() { + out.push(constant::FLOAT); + out.extend_from_slice(&number.to_le_bytes()); + return Ok(()); + } + if let Ok(character) = value.as_char() { + out.push(constant::CHAR); + put_uvarint(out, u32::from(character).into()); + return Ok(()); + } + if value.is_string() { + let text = value + .read_lock::() + .ok_or_else(|| value.type_name().to_string())?; + out.push(constant::STRING); + put_str(out, text.as_str()); + return Ok(()); + } + if value.is_array() { + let array = value + .read_lock::() + .ok_or_else(|| value.type_name().to_string())?; + out.push(constant::ARRAY); + put_uvarint(out, array.len() as u64); + for item in array.iter() { + put_constant(out, item)?; + } + return Ok(()); + } + if value.is_map() { + let map = value + .read_lock::() + .ok_or_else(|| value.type_name().to_string())?; + out.push(constant::MAP); + put_uvarint(out, map.len() as u64); + for (key, item) in map.iter() { + put_str(out, key.as_str()); + put_constant(out, item)?; + } + return Ok(()); + } + if value.is_blob() { + let blob = value + .read_lock::() + .ok_or_else(|| value.type_name().to_string())?; + out.push(constant::BLOB); + put_uvarint(out, blob.len() as u64); + out.extend_from_slice(&blob); + return Ok(()); + } + + // Behind the array and map checks because those are far more common, and + // after everything cheap because reaching one means two failed downcasts. + if let Some(range) = value.read_lock::>() { + out.push(constant::RANGE); + put_range(out, range.start, range.end); + return Ok(()); + } + if let Some(range) = value.read_lock::>() { + out.push(constant::RANGE_INCLUSIVE); + put_range(out, *range.start(), *range.end()); + return Ok(()); + } + + Err(value.type_name().to_string()) +} + +/// `INT` widens to `i64` on the wire; the fingerprint is what stops a reader +/// narrowing it back to something else. +fn put_range(out: &mut Vec, start: INT, end: INT) { + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(start)); + #[allow(clippy::useless_conversion)] + put_ivarint(out, i64::from(end)); +} diff --git a/src/grain/mod.rs b/src/grain/mod.rs new file mode 100644 index 000000000..aae648307 --- /dev/null +++ b/src/grain/mod.rs @@ -0,0 +1,41 @@ +//! A bytecode VM for Rhai. +//! +//! Rhai evaluates by walking its AST, which the parser allocates a node at a +//! time — so holding a script costs in proportion to how much program it is, +//! and the parser's peak is higher again than what it settles at. That is what +//! caps script size on a small target long before anything else does. +//! rhaigrain compiles the tree to a flat instruction stream that can be +//! produced elsewhere and loaded without a parser. +//! +//! `tests/allocation.rs` measures both ends of that with a tracking allocator. +//! +//! Execution reuses the host `Engine`: `Dynamic` stays the value type and every +//! registered function is dispatched by rhai itself. Only control flow, local +//! variable access and operator fast paths are reimplemented. +//! +//! A program that has been lowered all the way through can be written out with +//! [`Program::write`] and read back with [`Program::read`] — see [`format`]. +//! That is the artifact the device loads, and the reason the tree never has to +//! exist there. +//! +//! Coverage is total from the start, by construction rather than by effort. +//! Anything the compiler cannot yet lower is kept as an AST fragment and handed +//! back to rhai's walker through [`bytecode::Op::EvalAst`], so a `Program` +//! always means the same thing as the `AST` it came from. Progress is measured +//! by [`Program::residual_count`] falling, not by constructs becoming legal. + +// A VM that runs untrusted bytecode has no business containing any, and saying +// so here makes it the compiler's problem rather than a promise. `crates/ +// rhaigrain-pos` declares the same. +#![forbid(unsafe_code)] + +pub mod bytecode; +mod compile; +pub mod format; +pub mod pos; +mod program; +mod vm; + +pub use compile::Compiler; +pub use program::Program; +pub use vm::Vm; diff --git a/src/grain/pos/mod.rs b/src/grain/pos/mod.rs new file mode 100644 index 000000000..54093db3a --- /dev/null +++ b/src/grain/pos/mod.rs @@ -0,0 +1,321 @@ +//! Turning a rhaigrain instruction address back into a source position, +//! without `std`. +//! +//! A rhaigrain artifact carries no positions in its instruction stream. They +//! live in a separate table, so a device can be shipped the bytecode alone and +//! the table kept on the host that compiled it — which is the point, because a +//! position on every instruction is a large fraction of the artifact and is +//! never read unless something fails. +//! +//! What comes back is a [`Site`]: a line and a column as plain numbers. This +//! crate does not know what a `rhai::Position` is, and deliberately so, since +//! the reason to keep the table on-device is to report an error without +//! linking the compiler that produced it. +//! +//! ## Where this sits +//! +//! It resolves the first hop only. A script that was minified before it was +//! compiled needs a second: the [`Site`] is a position in the *minified* +//! source, and a Source Map v3 resolver such as `rhaiper-map` takes it the rest +//! of the way to the original. +//! +//! ```text +//! instruction address --[this crate]--> minified line:col --[a source map]--> original +//! ``` +//! +//! ## Table format +//! +//! ```text +//! varint entry count +//! per entry: varint address delta from the previous entry +//! varint line, 1-based (0 means the site is unknown) +//! varint column, 1-based (0 means the start of a line) +//! ``` +//! +//! Entries are sorted by address, so a lookup walks until it reaches or passes +//! the one it wants and stops. That is a linear scan, which is the right shape +//! for something only consulted when a program has already failed. + +pub mod varint; + +/// A place in a source file. +/// +/// Both fields follow rhai's own convention: `line` counts from 1, and +/// `column` counts characters from 1 with 0 meaning the start of a line. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Site { + /// 1-based line. + pub line: u32, + /// 1-based character column; 0 is the start of a line. + pub column: u32, +} + +/// Why a table could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + /// The table ends mid-entry. + Truncated, + /// A number too wide for the field it was read into. + Overflow, + /// Addresses are not strictly ascending, so the scan's early exit would be + /// wrong and a lookup could silently return the wrong site. + OutOfOrder { at: u32 }, + /// Bytes remain after the last entry, so this is not the table it claims. + TrailingBytes { count: usize }, +} + +impl From for Error { + fn from(err: varint::Error) -> Self { + match err { + varint::Error::Truncated => Self::Truncated, + varint::Error::Overflow => Self::Overflow, + } + } +} + +/// The site recorded for instruction `address`, if the table has one. +/// +/// `None` means the instruction has no recorded site — not that the table is +/// broken. Most instructions have none: only those that can raise an error +/// against a place in the source are worth recording. +/// +/// A malformed table also reads as `None`. Resolving happens while reporting +/// an error, and failing to resolve must not replace the error being reported; +/// call [`check`] at load time to find out whether a table is sound. +#[must_use] +pub fn resolve(table: &[u8], address: u32) -> Option { + let mut at = 0usize; + let count = varint::u32(table, &mut at).ok()?; + + let mut current = 0u32; + for _ in 0..count { + current = current.checked_add(varint::u32(table, &mut at).ok()?)?; + let line = varint::u32(table, &mut at).ok()?; + let column = varint::u32(table, &mut at).ok()?; + + if current == address { + return (line != 0).then_some(Site { line, column }); + } + // Ascending, so nothing past here can match. + if current > address { + return None; + } + } + + None +} + +/// How many entries the table holds, without decoding them. +/// +/// # Errors +/// +/// [`Error::Truncated`] or [`Error::Overflow`] if the count itself is unreadable. +pub fn count(table: &[u8]) -> Result { + let mut at = 0usize; + Ok(varint::u32(table, &mut at)?) +} + +/// Check that a table is well formed, so [`resolve`] can afford not to. +/// +/// # Errors +/// +/// Names the first thing wrong: a truncated entry, a number too wide, an +/// address that does not ascend, or bytes past the end. +pub fn check(table: &[u8]) -> Result<(), Error> { + let mut at = 0usize; + let count = varint::u32(table, &mut at)?; + + let mut previous: Option = None; + for _ in 0..count { + let delta = varint::u32(table, &mut at)?; + let address = match previous { + None => delta, + // A zero delta repeats the previous address, which would shadow it. + Some(previous) if delta == 0 => return Err(Error::OutOfOrder { at: previous }), + Some(previous) => previous.checked_add(delta).ok_or(Error::Overflow)?, + }; + let _line = varint::u32(table, &mut at)?; + let _column = varint::u32(table, &mut at)?; + previous = Some(address); + } + + if at != table.len() { + return Err(Error::TrailingBytes { + count: table.len() - at, + }); + } + + Ok(()) +} + +/// Build a table from sites in ascending address order. +/// +/// Entries with a zero line are dropped: they carry no information, and +/// keeping them would cost bytes on a table whose whole purpose is to be small +/// enough to leave behind. +/// +/// # Panics +/// +/// Panics if the addresses are not strictly ascending. That is a caller bug — +/// a compiler emits in address order by construction — and producing a table +/// that silently resolves wrong is worse than not producing one. +#[must_use] +pub fn encode(sites: impl IntoIterator) -> alloc::vec::Vec { + let kept: alloc::vec::Vec<_> = sites + .into_iter() + .filter(|(_, site)| site.line != 0) + .collect(); + + let mut out = alloc::vec::Vec::new(); + varint::put_u64(&mut out, kept.len() as u64); + + let mut previous = 0u32; + for (index, (address, site)) in kept.iter().enumerate() { + if index > 0 { + assert!( + *address > previous, + "table addresses must strictly ascend: {address} follows {previous}", + ); + } + varint::put_u64(&mut out, u64::from(address - previous)); + varint::put_u64(&mut out, u64::from(site.line)); + varint::put_u64(&mut out, u64::from(site.column)); + previous = *address; + } + + out +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use alloc::vec::Vec; + + fn site(line: u32, column: u32) -> Site { + Site { line, column } + } + + fn table() -> Vec { + encode(vec![ + (0, site(1, 5)), + (3, site(1, 11)), + (40, site(7, 2)), + (41, site(9, 0)), + ]) + } + + #[test] + fn a_recorded_address_resolves_to_its_site() { + let table = table(); + assert_eq!(resolve(&table, 0), Some(site(1, 5))); + assert_eq!(resolve(&table, 3), Some(site(1, 11))); + assert_eq!(resolve(&table, 40), Some(site(7, 2))); + } + + /// Column 0 is the start of a line, which is a real position and must not + /// be confused with an absent one. + #[test] + fn the_start_of_a_line_is_a_position() { + assert_eq!(resolve(&table(), 41), Some(site(9, 0))); + } + + #[test] + fn an_address_with_no_site_resolves_to_nothing() { + let table = table(); + assert_eq!(resolve(&table, 1), None, "between two entries"); + assert_eq!(resolve(&table, 99), None, "past the last entry"); + } + + #[test] + fn an_empty_table_resolves_nothing_and_is_sound() { + let table = encode(Vec::new()); + assert_eq!(check(&table), Ok(())); + assert_eq!(count(&table), Ok(0)); + assert_eq!(resolve(&table, 0), None); + } + + /// A site with no line carries nothing, and the table exists to be small. + #[test] + fn sites_with_no_line_are_not_stored() { + let table = encode(vec![(0, site(1, 1)), (1, site(0, 0)), (2, site(3, 3))]); + assert_eq!(count(&table), Ok(2)); + assert_eq!(resolve(&table, 1), None); + assert_eq!(resolve(&table, 2), Some(site(3, 3))); + } + + /// Deltas are the reason the table is worth encoding rather than storing + /// flat: consecutive instructions cost one byte of address each. + #[test] + fn a_dense_run_costs_three_bytes_an_entry() { + let dense: Vec<_> = (0..100).map(|pc| (pc, site(1, 1))).collect(); + let table = encode(dense); + assert_eq!(table.len(), 1 + 100 * 3); + } + + #[test] + fn a_sound_table_passes_its_own_check() { + assert_eq!(check(&table()), Ok(())); + } + + #[test] + fn a_truncated_table_is_named_as_such() { + let table = table(); + for cut in 1..table.len() { + assert!( + check(&table[..cut]).is_err(), + "a {cut}-byte prefix passed the check", + ); + } + } + + #[test] + fn a_repeated_address_is_refused() { + // Hand-built, since `encode` will not produce one. + let mut bytes = Vec::new(); + varint::put_u64(&mut bytes, 2); + for _ in 0..2 { + varint::put_u64(&mut bytes, 0); // delta + varint::put_u64(&mut bytes, 1); // line + varint::put_u64(&mut bytes, 1); // column + } + assert_eq!(check(&bytes), Err(Error::OutOfOrder { at: 0 })); + } + + #[test] + fn bytes_past_the_last_entry_are_refused() { + let mut bytes = table(); + bytes.push(0); + assert_eq!(check(&bytes), Err(Error::TrailingBytes { count: 1 })); + } + + /// Resolution runs while an error is being reported. Whatever the table + /// says, it may not become the failure. + #[test] + fn no_byte_string_can_make_resolution_panic() { + let table = table(); + for index in 0..table.len() { + for bit in 0..8 { + let mut corrupt = table.clone(); + corrupt[index] ^= 1 << bit; + for address in 0..64 { + let _ = resolve(&corrupt, address); + } + let _ = check(&corrupt); + } + } + + for junk in [&b""[..], &[0xff][..], &[0xff; 32][..], &[0x80; 12][..]] { + for address in 0..8 { + let _ = resolve(junk, address); + } + let _ = check(junk); + } + } + + #[test] + #[should_panic(expected = "must strictly ascend")] + fn encoding_out_of_order_sites_is_a_caller_bug() { + let _ = encode(vec![(5, site(1, 1)), (2, site(1, 1))]); + } +} diff --git a/src/grain/pos/varint.rs b/src/grain/pos/varint.rs new file mode 100644 index 000000000..2e1c323cd --- /dev/null +++ b/src/grain/pos/varint.rs @@ -0,0 +1,136 @@ +//! LEB128, the encoding everything in a rhaigrain artifact is counted in. +//! +//! Small numbers dominate a chunk — pool indices, jump targets, line numbers — +//! and almost all of them fit in one byte. Signed values are zigzagged first, +//! so a small negative costs no more than a small positive. +//! +//! Decoding takes the cursor by reference rather than owning one, so the same +//! functions serve a reader that tracks its own errors and a resolver that has +//! nothing but a byte slice. + +/// Why a varint could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + /// The bytes ran out mid-value. + Truncated, + /// A value wider than 64 bits, or a run of continuation bytes that never + /// ends. Refusing both is what stops a corrupt table from spinning. + Overflow, +} + +/// Read an unsigned varint, advancing `at`. +/// +/// # Errors +/// +/// [`Error::Truncated`] if the slice ends first, [`Error::Overflow`] if the +/// value does not fit in 64 bits. +pub fn u64(bytes: &[u8], at: &mut usize) -> Result { + let mut value = 0u64; + for shift in (0..64).step_by(7) { + let byte = *bytes.get(*at).ok_or(Error::Truncated)?; + *at += 1; + value |= ((byte & 0x7f) as u64) + .checked_shl(shift) + .ok_or(Error::Overflow)?; + if byte & 0x80 == 0 { + return Ok(value); + } + } + Err(Error::Overflow) +} + +/// Read a zigzagged signed varint, advancing `at`. +/// +/// # Errors +/// +/// As [`u64`]. +pub fn i64(bytes: &[u8], at: &mut usize) -> Result { + let raw = u64(bytes, at)?; + Ok(((raw >> 1) as i64) ^ -((raw & 1) as i64)) +} + +/// Read an unsigned varint that must fit a `u32`, advancing `at`. +/// +/// # Errors +/// +/// As [`u64`], plus [`Error::Overflow`] if the value is too wide for the field. +pub fn u32(bytes: &[u8], at: &mut usize) -> Result { + core::convert::TryFrom::try_from(u64(bytes, at)?).map_err(|_| Error::Overflow) +} + +/// Append an unsigned varint. +pub fn put_u64(out: &mut alloc::vec::Vec, mut value: u64) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +/// Append a zigzagged signed varint. +pub fn put_i64(out: &mut alloc::vec::Vec, value: i64) { + put_u64(out, ((value << 1) ^ (value >> 63)) as u64); +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec::Vec; + + #[test] + fn unsigned_values_round_trip_at_the_edges() { + for value in [0u64, 1, 127, 128, 16383, 16384, u32::MAX as u64, u64::MAX] { + let mut buf = Vec::new(); + put_u64(&mut buf, value); + let mut at = 0; + assert_eq!(u64(&buf, &mut at), Ok(value), "at {value}"); + assert_eq!(at, buf.len(), "the reader must consume exactly what was written"); + } + } + + #[test] + fn signed_values_round_trip_across_zero() { + for value in [0i64, -1, 1, -64, 63, i32::MIN as i64, i64::MIN, i64::MAX] { + let mut buf = Vec::new(); + put_i64(&mut buf, value); + let mut at = 0; + assert_eq!(i64(&buf, &mut at), Ok(value), "at {value}"); + } + } + + /// The encoding only pays for itself if the common case is one byte. + #[test] + fn values_under_128_cost_one_byte() { + let mut buf = Vec::new(); + put_u64(&mut buf, 127); + assert_eq!(buf.len(), 1); + put_i64(&mut buf, -64); + assert_eq!(buf.len(), 2); + } + + #[test] + fn a_run_of_continuation_bytes_terminates() { + let mut at = 0; + assert_eq!(u64(&[0xff; 64], &mut at), Err(Error::Overflow)); + } + + #[test] + fn running_out_of_bytes_is_an_error_not_a_panic() { + let mut at = 0; + assert_eq!(u64(&[], &mut at), Err(Error::Truncated)); + let mut at = 0; + assert_eq!(u64(&[0x80], &mut at), Err(Error::Truncated)); + } + + #[test] + fn a_value_too_wide_for_its_field_is_refused() { + let mut buf = Vec::new(); + put_u64(&mut buf, u64::from(u32::MAX) + 1); + let mut at = 0; + assert_eq!(u32(&buf, &mut at), Err(Error::Overflow)); + } +} diff --git a/src/grain/program.rs b/src/grain/program.rs new file mode 100644 index 000000000..3170334df --- /dev/null +++ b/src/grain/program.rs @@ -0,0 +1,590 @@ +use crate::ast::{ASTFlags, ASTNode}; +#[cfg(not(feature = "no_module"))] +use crate::module_resolvers::StaticModuleResolver; +use crate::{ast::Expr, ast::Stmt, tokenizer::Token, Dynamic, ImmutableString, Module, Shared}; + +use crate::grain::bytecode::{ + AssignOp, Chain, Chunk, Code, Pools, Positions, Strings, Switch, TableError, +}; + +/// rhai's own `SharedModule`, which it does not re-export. +pub(crate) type SharedModule = Shared; + +/// A program a native function can be handed a way back into. +/// +/// [`Vm::run_with_callbacks`](crate::Vm::run_with_callbacks) registers one +/// wrapper per compiled function, and rhai requires a registered function to be +/// `'static` — so the program cannot still be borrowing an artifact, and the +/// wrappers have to share ownership of it rather than borrow it. +pub type SharedProgram = Shared>; + +/// A compiled script, ready to run against an `Engine`. +/// +/// Owns everything execution needs that is not the `Engine` itself, so the +/// original `AST` can be dropped after compiling. On a small target that is the +/// whole point: the tree is the part whose cost scales with the program. +/// +/// The lifetime is the artifact's. A program read from bytes borrows its +/// instructions from them and allocates only its pools, which are bounded by +/// the distinct constants and names a script actually mentions rather than by +/// how long it is. [`Program::into_owned`] cuts the tie when that is wanted. +/// +/// `residuals` is the exception, and the reason a `Program` is not always +/// serializable. Fragments rhai's walker still has to evaluate are held as real +/// `Expr` trees, which is precisely the allocation we are trying to remove. The +/// artifact format refuses to write a `Program` that has any, so nothing +/// reaching a device can depend on them. +/// One compiled script function. +/// +/// Called by [`Op::Call`](crate::bytecode::Op::Call) directly, without going +/// through rhai's dispatch: the name is already an index into the same pool the +/// call site used, so matching one is two integer comparisons rather than a +/// hash and a module walk. +#[derive(Debug, Clone)] +pub struct Function { + /// Index into the name pool. + pub name: u32, + /// Parameter names, in order, as name-pool indices. They become the + /// callee's first locals, which is what makes them slot 0 upwards. + pub params: Vec, + pub chunk: Chunk, +} + +pub struct Program<'a> { + /// Every chunk's instructions, concatenated: main first, then each + /// function. One buffer means one position table and one instruction + /// address, so a device that fails reports a single number. + code: Code<'a>, + + main: Chunk, + + /// Script functions compiled to chunks. Empty when none were compiled, + /// which is when rhai's own versions are carried in `lib` instead. + functions: Vec, + + /// The deepest chunk's operand-stack need, cached. + max_stack: u16, + + /// Whether the program can hand a function pointer to something that + /// might call it back. See [`Program::makes_fn_pointers`]. + makes_fn_pointers: bool, + + /// Where each instruction came from, or [`Positions::Stripped`]. + /// + /// Separable on purpose: a device is shipped the code and the host keeps + /// the table, so an error arrives as an instruction address and is resolved + /// where the source is. See [`crate::bytecode::Positions`]. + positions: Positions, + + residuals: Vec, + + /// Values `Op::Const` indexes. Deduplicated, so a constant repeated across + /// the script is stored once. + consts: Vec, + + /// Every name the program mentions, as one borrowed blob. + /// + /// Nothing needs a `String` of its own. Call names, operators, getters and + /// property keys go to rhai as `&str`; a `Scope` entry name goes in as an + /// `Identifier`, which is a `SmartString` and keeps a short name inline + /// rather than on the heap. So the whole table is two allocations — the + /// blob and the spans — and neither grows with how many names there are. + names: Strings<'a>, + + /// Operator tokens the built-in lookup keys on. A `Token` does not fit an + /// operand, and one script uses a handful of distinct operators however + /// many times it mentions them. + tokens: Vec, + + /// What each `x op= y` site needs, for the same reason. + assign_ops: Vec, + + /// The steps of each `a.b[i].c`. Out of the instruction stream because a + /// chain is one instruction however many steps it has. + chains: Vec, + + /// One dispatch table per `switch`, for the same reason. + /// + /// Case hashes are rhai's, and rhai's hasher is seeded per process unless + /// the host says otherwise — so an artifact carrying any of these carries + /// a [`probe`](crate::bytecode::probe) too, and refuses to load against a + /// hasher that would disagree with it. + switches: Vec, + + /// Script functions the compiler did not lower, as rhai's own library, so + /// a fragment can still call one the ordinary way. + /// + /// `None` when the script declared none, which is every program that came + /// from an artifact. An empty `Module` is 264 bytes and a reference count, + /// which is a fifth of what loading a small program retains — worth not + /// allocating for something nothing will look in. + lib: Option, + + /// Reinstated on the runtime state at each run, mirroring + /// `Engine::eval_ast_with_scope_raw`, so `import` resolves as it would have. + #[cfg(not(feature = "no_module"))] + resolver: Option>, + + /// Names the script in error messages and `NativeCallContext::call_source`. + source: Option, +} + +/// A summary rather than a dump: the library alone would render every script +/// function's whole AST, which is never what someone printing a `Program` +/// wants to read. +impl core::fmt::Debug for Program<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Program") + .field("source", &self.source) + .field("bytes", &self.code.len()) + .field("max_stack", &self.main.max_stack()) + .field("consts", &self.consts.len()) + .field("names", &self.names.len()) + .field("residuals", &self.residuals.len()) + .field("compiled_fns", &self.functions.len()) + .field( + "walked_fns", + &self.lib.as_ref().map_or(0, |lib| lib.count().1), + ) + .field("positions", &!self.positions.is_stripped()) + .finish() + } +} + +/// What a script author would call the construct at this node, for the +/// constructs the compiler does not lower yet. +/// +/// Only the ones worth naming: an author can act on "switch at line 42", not +/// on "Expr::Dot". Anything else falls through to the generic message. +fn unsupported_kind(node: &ASTNode) -> Option<&'static str> { + Some(match node { + ASTNode::Stmt(stmt) => match stmt { + Stmt::Switch(..) => "switch", + Stmt::For(..) => "for", + Stmt::TryCatch(..) => "try/catch", + #[cfg(not(feature = "no_module"))] + Stmt::Import(..) => "import", + #[cfg(not(feature = "no_module"))] + Stmt::Export(..) => "export", + #[cfg(not(feature = "no_closure"))] + Stmt::Share(..) => "a closure capture", + Stmt::Return(_, flags, ..) if flags.intersects(ASTFlags::BREAK) => "throw", + _ => return None, + }, + ASTNode::Expr(expr) => match expr { + Expr::InterpolatedString(..) => "string interpolation", + Expr::Custom(..) => "custom syntax", + Expr::Map(..) => "a non-constant map literal", + _ => return None, + }, + }) +} + +fn node_position(node: &ASTNode) -> rhai::Position { + match node { + ASTNode::Stmt(stmt) => stmt.position(), + ASTNode::Expr(expr) => expr.start_position(), + } +} + +/// Whether any instruction in `code` produces a function pointer. +/// +/// Read off the bytes rather than tracked while lowering, because a program +/// read from an artifact has no lowering to have tracked it — and the answer +/// has to be the same either way or one of the two paths silently loses its +/// callbacks. +/// +/// Deliberately over-broad: it says yes to a pointer that is only ever called +/// directly. Narrowing it would mean deciding where a pointer *goes*, which is +/// a dataflow question over values that outlive the instruction that made +/// them. Being wrong the other way loses a call at run time. +fn makes_fn_pointers(code: &[u8]) -> bool { + use crate::grain::bytecode::code::tag; + + crate::grain::bytecode::disassemble(code) + .any(|(at, ..)| matches!(code[at], tag::MAKE_CLOSURE | tag::MAKE_FN_PTR | tag::CURRY)) +} + +/// Everything a program holds besides its code, gathered so the constructor +/// does not take ten positional arguments. +pub(crate) struct Parts<'a> { + pub positions: Positions, + pub residuals: Vec, + pub consts: Vec, + pub names: Strings<'a>, + pub tokens: Vec, + pub assign_ops: Vec, + pub chains: Vec, + pub switches: Vec, + pub lib: Option, + #[cfg(not(feature = "no_module"))] + pub resolver: Option>, + pub source: Option, +} + +impl<'a> Program<'a> { + pub(crate) fn new( + code: Code<'a>, + main: Chunk, + functions: Vec, + parts: Parts<'a>, + ) -> Self { + let makes_fn_pointers = makes_fn_pointers(&code); + let mut program = Self { + code, + main, + functions, + max_stack: 0, + makes_fn_pointers, + positions: parts.positions, + residuals: parts.residuals, + consts: parts.consts, + names: parts.names, + tokens: parts.tokens, + assign_ops: parts.assign_ops, + chains: parts.chains, + switches: parts.switches, + lib: parts.lib, + #[cfg(not(feature = "no_module"))] + resolver: parts.resolver, + source: parts.source, + }; + program.recompute_max_stack(); + program + } + + /// Copy the borrowed instructions, so this program outlives the artifact it + /// was read from. + /// + /// The opposite of the point, and only worth it when the buffer has to go. + #[must_use] + pub fn into_owned(self) -> Program<'static> { + Program { + code: Code::Owned(self.code.into_owned()), + main: self.main, + functions: self.functions, + max_stack: self.max_stack, + makes_fn_pointers: self.makes_fn_pointers, + positions: self.positions, + residuals: self.residuals, + consts: self.consts, + names: self.names.into_owned(), + tokens: self.tokens, + assign_ops: self.assign_ops, + chains: self.chains, + switches: self.switches, + lib: self.lib, + #[cfg(not(feature = "no_module"))] + resolver: self.resolver, + source: self.source, + } + } + + /// Give up the artifact and share the program, so a native can be handed a + /// way back into it. + /// + /// What [`Vm::run_with_callbacks`](crate::Vm::run_with_callbacks) takes. + /// Worth the copy only when [`makes_fn_pointers`](Self::makes_fn_pointers) + /// says a pointer can escape. + #[must_use] + pub fn into_shared(self) -> SharedProgram { + Shared::new(self.into_owned()) + } + + /// Check the chunk is internally consistent, returning the stack high water + /// it measured. + /// + /// Cheap enough to run on every compile, and the gate an artifact loaded + /// from a wire has to pass before the VM will touch it. + pub fn verify(&self) -> Result, crate::grain::bytecode::VerifyError> { + crate::grain::bytecode::verify(&self.code, &self.chunks(), self.pools()) + } + + /// Every chunk, main first, in the order they sit in the code. + fn chunks(&self) -> Vec { + core::iter::once(self.main) + .chain(self.functions.iter().map(|f| f.chunk)) + .collect() + } + + pub(crate) fn pools(&self) -> Pools<'_> { + Pools { + consts: self.consts.len(), + names: self.names.len(), + tokens: self.tokens.len(), + assign_ops: self.assign_ops.len(), + residuals: self.residuals.len(), + chains: &self.chains, + switches: &self.switches, + } + } + + /// Replace the compiler's upper-bound stack estimate with the verified high + /// water, so the VM reserves what the chunk uses rather than one slot per + /// instruction. + /// + /// A chunk that does not verify keeps its estimate: the VM is still safe + /// with a value that is too large, and [`Program::verify`] is where the + /// real failure should surface. + pub(crate) fn tighten_stack(&mut self) { + let Ok(high_water) = self.verify() else { + return; + }; + let mut measured = high_water.into_iter(); + if let Some(main) = measured.next() { + self.main.set_max_stack(main); + } + for (function, high_water) in self.functions.iter_mut().zip(measured) { + function.chunk.set_max_stack(high_water); + } + self.recompute_max_stack(); + } + + #[must_use] + pub fn code(&self) -> &[u8] { + &self.code + } + + #[must_use] + pub fn functions(&self) -> &[Function] { + &self.functions + } + + /// The compiled function a call site resolves to, if there is one. + /// + /// Name and arity only, matching how rhai keys script functions. The name + /// is an index into the pool the call site also indexes, so equal names + /// have equal indices and this is two integer comparisons. + pub(crate) fn function(&self, name: u32, argc: usize) -> Option<&Function> { + self.functions + .iter() + .find(|f| f.name == name && f.params.len() == argc) + } + + /// The compiled function a *pointer* resolves to. + /// + /// By name rather than by pool index, because a `FnPtr` carries a string — + /// it may have been built from one at run time. A linear scan, which at + /// these sizes beats a map and keeps the common indexed lookup untouched. + pub(crate) fn function_named(&self, name: &str, argc: usize) -> Option<&Function> { + self.functions + .iter() + .find(|f| f.params.len() == argc && self.name(f.name) == Some(name)) + } + + /// Whether this program can hand a function pointer to something that + /// might call it back. + /// + /// A compiled function lives in this program's own table and nowhere rhai + /// can see, so a native that calls a pointer — `map`, `filter` — cannot + /// reach one. Making it reachable means registering a wrapper, and rhai + /// requires a registered function to be `'static`, so the wrapper has to + /// own the program: [`Program::into_owned`] first, at the cost of the + /// borrowed-from-the-artifact loading that is the point of the format. + /// + /// This is how a host decides whether to pay that, without having to read + /// the script. False is the common answer and costs nothing. + #[must_use] + pub fn makes_fn_pointers(&self) -> bool { + self.makes_fn_pointers + } + + /// How much operand stack the deepest chunk needs. + /// + /// One reservation serves every frame, because a call pushes its operands + /// above the caller's rather than starting a stack of its own. Cached + /// rather than recomputed, since entering a frame reads it and entering a + /// frame is what a call does. + #[must_use] + pub fn max_stack(&self) -> u16 { + self.max_stack + } + + fn recompute_max_stack(&mut self) { + self.max_stack = self + .functions + .iter() + .map(|f| f.chunk.max_stack()) + .chain(core::iter::once(self.main.max_stack())) + .max() + .unwrap_or(0); + } + + pub(crate) fn constant(&self, index: u32) -> Option<&Dynamic> { + self.consts.get(index as usize) + } + + /// A name, borrowed from the artifact. Never allocates. + pub(crate) fn name(&self, index: u32) -> Option<&str> { + self.names.get(index) + } + + pub(crate) fn token(&self, index: u32) -> Option<&Token> { + self.tokens.get(index as usize) + } + + pub(crate) fn assign_op(&self, index: u32) -> Option<&AssignOp> { + self.assign_ops.get(index as usize) + } + + pub(crate) fn chain(&self, index: u32) -> Option<&Chain> { + self.chains.get(index as usize) + } + + pub(crate) fn chains(&self) -> &[Chain] { + &self.chains + } + + pub(crate) fn switch(&self, index: u32) -> Option<&Switch> { + self.switches.get(index as usize) + } + + /// The dispatch tables [`Op::Switch`](crate::bytecode::Op::Switch) indexes. + /// + /// Public because a disassembly that leaves them out is misleading: a + /// switch's arms are reached only from its table, so without it they read + /// as unreachable code. + #[must_use] + pub fn switches(&self) -> &[Switch] { + &self.switches + } + + /// Where instruction `pc` came from, or `Position::NONE` if the table was + /// stripped or has nothing for it. + #[must_use] + pub fn position(&self, pc: usize) -> rhai::Position { + self.positions.get(pc) + } + + #[must_use] + pub fn positions(&self) -> &Positions { + &self.positions + } + + /// Drop the position table, returning it in its compact wire form. + /// + /// This is the separation the debug layer exists for: ship the program to + /// the device and keep what comes back here. Errors then arrive with no + /// position, and [`rhaigrain_pos::resolve`] turns the failing instruction + /// address back into one — see [`Program::attach_positions`] for the + /// inverse. + pub fn strip_positions(&mut self) -> Vec { + let table = self.positions.to_table(); + self.positions = Positions::Stripped; + table + } + + /// Put a stripped table back, so this program reports positions again. + /// + /// # Errors + /// + /// Refuses a malformed table, and refuses one whose addresses do not fit + /// this chunk — attaching another program's table would misreport every + /// error rather than reporting none, which is strictly worse. + pub fn attach_positions(&mut self, table: &[u8]) -> Result<(), TableError> { + self.positions = Positions::from_table(table, self.code.len())?; + Ok(()) + } + + pub(crate) fn consts(&self) -> &[Dynamic] { + &self.consts + } + + pub(crate) fn names(&self) -> &Strings<'a> { + &self.names + } + + pub(crate) fn tokens(&self) -> &[Token] { + &self.tokens + } + + pub(crate) fn assign_ops(&self) -> &[AssignOp] { + &self.assign_ops + } + + #[must_use] + pub fn main(&self) -> &Chunk { + &self.main + } + + /// How many fragments rhai's walker still evaluates. + /// + /// Non-zero is the reason a program cannot yet be serialized. As a measure + /// of progress it is misleading on its own: lowering a statement often + /// splits one fragment into several smaller ones, so the count rises while + /// the work left shrinks. Use [`Program::residual_nodes`] for that. + #[must_use] + pub fn residual_count(&self) -> usize { + self.residuals.len() + } + + /// How many AST nodes are still inside fragments. + /// + /// This is the progress metric that only falls: it counts the tree that has + /// to survive into the artifact. + #[must_use] + pub fn residual_nodes(&self) -> usize { + let mut nodes = 0; + let path = &mut Vec::new(); + for residual in &self.residuals { + residual.walk(path, &mut |_| { + nodes += 1; + true + }); + } + nodes + } + + pub(crate) fn residual(&self, index: u32) -> Option<&Expr> { + self.residuals.get(index as usize) + } + + /// The construct that stopped this program being written, and where. + /// + /// A count of fragments is not something anyone can act on. This names the + /// first thing the compiler could not lower, so a validator can reject an + /// upload with the line to go and look at — which is what makes falling + /// back to shipping source a decision rather than a mystery. + #[must_use] + pub fn first_unsupported(&self) -> Option<(&'static str, rhai::Position)> { + let path = &mut Vec::new(); + let mut found: Option<(&'static str, rhai::Position)> = None; + + for residual in &self.residuals { + residual.walk(path, &mut |path| { + if found.is_some() { + return false; + } + if let Some(name) = path.last().and_then(unsupported_kind) { + found = Some((name, node_position(path.last().expect("just matched")))); + return false; + } + true + }); + if found.is_some() { + break; + } + } + + // A fragment made of nothing this recognises is still a fragment, so + // say so rather than reporting nothing wrong. + found.or_else(|| { + self.residuals + .first() + .map(|expr| ("an unlowered expression", expr.start_position())) + }) + } + + pub(crate) fn lib(&self) -> Option<&SharedModule> { + self.lib.as_ref() + } + + #[cfg(not(feature = "no_module"))] + pub(crate) fn resolver(&self) -> Option<&Shared> { + self.resolver.as_ref() + } + + pub(crate) fn source(&self) -> Option<&ImmutableString> { + self.source.as_ref() + } +} diff --git a/src/grain/vm/callback.rs b/src/grain/vm/callback.rs new file mode 100644 index 000000000..d718011be --- /dev/null +++ b/src/grain/vm/callback.rs @@ -0,0 +1,130 @@ +//! Reaching a compiled chunk from inside a native function. +//! +//! `[1, 2, 3].map(|x| x * 2)` leaves this VM in the middle of a call. `map` is +//! rhai's, and the pointer it calls back is resolved by rhai's dispatch, which +//! looks in `global.lib` and the engine's modules. Our chunks are in neither: +//! [`Op::Call`](crate::bytecode::Op::Call) finds one by a name *index* that +//! only the compiler and the call site share, and a `FnPtr` carries a string. +//! +//! So a program that hands a pointer out registers one native wrapper per +//! compiled function for the length of the run. Direct dispatch is untouched — +//! this is somewhere for rhai to look, not somewhere we look. +//! +//! # What being a native costs +//! +//! Rhai reaches its own closures through a `Fn*` pointer carrying the body, and +//! that shortcut is what these wrappers cannot have. Two consequences, both +//! measured rather than assumed: +//! +//! * **Argument order, for a capturing closure called by a native that binds +//! `this`.** A capture is a curried value, and `_call_with_extra_args` +//! (`types/fn_ptr.rs:573`) tries `[this] ++ curry ++ args` first for anything +//! that is not a `Fn*` pointer. That shape is never right, and it is what a +//! wrapper answers to. Stock rhai does the same thing to its own name-only +//! pointers — `stock_rhai_does_the_same_to_its_own_native_pointers` in +//! `tests/callback.rs` reproduces it with none of this involved — so the fix +//! is not here; it is upstream, or in not currying captures at all. +//! * **Speed, and call budget.** Rhai resolves a wrapper by name and type on +//! every element, from a cache it builds fresh per crossing, where its own +//! pointer skips resolution entirely. `native callbacks` in +//! `examples/bench.rs` measures 0.34x — the one case the VM loses — and the +//! two extra dispatch layers cost 5 call levels per crossing against the +//! walker's 2. +//! +//! Neither touches a pointer called directly from compiled code, which is +//! `Op::CallFnPtr` and never comes through here. + +use std::any::TypeId; +use std::mem; + +use crate::{ + func::RhaiFunc, Dynamic, FuncRegistration, ImmutableString, Module, NativeCallContext, Shared, +}; + +use super::{malformed, Vm, VmResult}; +use crate::grain::program::SharedProgram; + +/// The most parameters a wrapper is registered for. +/// +/// A wrapper takes `Dynamic` throughout, and rhai only reaches a `Dynamic` +/// parameter by permuting the call's own argument types towards it — a search +/// it caps at `MAX_DYNAMIC_PARAMETERS`, 16 (`func/call.rs:235`). A wider +/// wrapper would be registered and never found, so it is left out rather than +/// silently dead. Direct dispatch has no such bound; this limits only what a +/// native can call back into. +const MAX_PARAMS: usize = 16; + +/// A wrapper per compiled function, for rhai to resolve a pointer against. +/// +/// Built once per run rather than cached on the program: the closures hold the +/// program, so anything the program held back would be a cycle. +pub(super) fn wrappers(program: &SharedProgram) -> Module { + let mut module = Module::new(); + + // What rhai reports as the source of a function it found here. Its own + // script library is the AST's, so this is the same string by the same + // route. + if let Some(source) = program.source() { + module.set_id(source.clone()); + } + + for function in program.functions() { + let arity = function.params.len(); + if arity > MAX_PARAMS { + continue; + } + let Some(name) = program.name(function.name) else { + continue; + }; + + let owner = program.clone(); + let called: ImmutableString = name.into(); + + // One closure for every arity, rather than the fixed-arity shapes + // `Module::set_native_fn` generates. `Dynamic` parameters throughout + // mean the types never have to line up, only the count. + let wrapper = move |context: Option, args: &mut [&mut Dynamic]| { + invoke(&owner, &called, context.as_ref(), args) + }; + + FuncRegistration::new(name) + .in_internal_namespace() + .set_into_module_raw( + &mut module, + vec![TypeId::of::(); arity], + RhaiFunc::Pure { + func: Shared::new(wrapper), + has_context: true, + is_pure: true, + is_volatile: false, + }, + ); + } + + module +} + +/// Run one chunk for a native that called back into us. +fn invoke( + program: &SharedProgram, + name: &str, + context: Option<&NativeCallContext>, + args: &mut [&mut Dynamic], +) -> VmResult { + // Registered with `has_context`, so rhai always supplies one. + let context = + context.ok_or_else(|| malformed("a callback wrapper was given no context".into()))?; + + // Taken rather than cloned, as every registered function does: the + // arguments are the caller's to give away, and it has already copied + // anything it still needs. + let values: Vec = args.iter_mut().map(|arg| mem::take(*arg)).collect(); + + Vm::reentrant(context).call_function( + program, + name, + values, + context.call_level(), + context.call_position(), + ) +} diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs new file mode 100644 index 000000000..081528564 --- /dev/null +++ b/src/grain/vm/mod.rs @@ -0,0 +1,2825 @@ +use core::mem; + +use crate::engine::{FN_IDX_GET, FN_IDX_SET}; +use crate::eval::calc_data_sizes; +use crate::func::{get_builtin_binary_op_fn, get_builtin_op_assignment_fn}; +use crate::packages::string_basic::print_with_func; +use crate::types::dynamic::DynamicWriteLock; +use crate::types::fn_ptr::FnPtrType; +use crate::{ + eval::Caches, eval::GlobalRuntimeState, Dynamic, Engine, EvalAltResult, EvalContext, + Expression, Scope, +}; +use crate::{ + Array, FnPtr, ImmutableString, Map, NativeCallContext, Position, ThinVec, FUNC_TO_STRING, INT, +}; + +mod callback; + +use crate::grain::bytecode::{code, AssignOp, Chain, Receiver, Root, Step, Tail}; +use crate::grain::program::{Program, SharedModule, SharedProgram}; + +/// rhai's own `RhaiResult`, which it does not re-export. +pub type VmResult = Result>; + +/// Whether a value is a shared cell. +/// +/// Sharing is how closures capture, so under `no_closure` there are no cells, +/// `Dynamic` has no `is_shared` to call, and the answer is a constant. The +/// opcodes that create and read cells are compiled out with it. +#[cfg(not(feature = "no_closure"))] +macro_rules! is_shared { + ($value:expr) => { + $value.is_shared() + }; +} +#[cfg(feature = "no_closure")] +macro_rules! is_shared { + ($value:expr) => {{ + let _ = &$value; + false + }}; +} + +/// A chunk that does not agree with itself — a slot past the end of the scope, +/// an index into a pool that has no such entry, an operand stack that ran dry. +/// +/// Reachable only through a compiler bug or a corrupt artifact, never through +/// anything a script can express. Verification turns most of these into load +/// time failures; the rest surface as runtime errors rather than panics, so a +/// bad chunk cannot take the host down. +/// Build a [`NativeCallContext`] from its parts. +/// +/// `NativeCallContext::new_with_all_fields` is the obvious spelling but is +/// `#[cfg(not(feature = "no_module"))]`. The `From` impl over the same five +/// fields is not gated and assigns exactly the same ones, so this works in +/// either configuration without a cfg of its own. +fn native_context<'a>( + engine: &'a Engine, + fn_name: &'a str, + source: Option<&'a str>, + global: &'a GlobalRuntimeState, + pos: Position, +) -> NativeCallContext<'a> { + NativeCallContext::from((engine, fn_name, source, global, pos)) +} + +/// Stamp the call site on an error that passes through a function boundary +/// unwrapped, as rhai does for exits and system exceptions +/// (`func/script.rs:134`). +fn reposition(mut err: Box, pos: Position) -> Box { + err.set_position(pos); + err +} + +/// Stamp the site on an error that arrived without one. +/// +/// Dispatch raises `ErrorFunctionNotFound` at `Position::NONE` and leaves +/// positioning to the caller, which has the expression that failed. Unlike +/// [`reposition`] this never overwrites a position the callee already set. +fn positioned(err: Box, pos: Position) -> Box { + if err.position().is_none() { + reposition(err, pos) + } else { + err + } +} + +/// Stamp the call site on anything a dispatched call came back with. +/// +/// This is `fill_position`, which rhai applies to the whole of +/// `exec_native_fn_call` — the callee not being found, the call being refused, +/// and the error a native *returned* alike (`func/call.rs:365`, `:406`, +/// `:413`). `call_fn_raw` has no position to give, so all of them arrive bare. +/// +/// What looks like a counter-example is not one: `1 / 0` reports +/// `ErrorArithmetic` with no position at all, because under `fast_operators` a +/// binary operator returns the built-in's error without going through here +/// (`func/call.rs:1798`). The VM's own fast path skips it for the same reason. +fn dispatch_failure(err: Box, pos: Position) -> Box { + positioned(err, pos) +} + +/// A scope entry, addressed the way whatever wants it was written. +/// +/// A slot always names one and a name may name nothing, which is the whole of +/// the difference between the two at run time — for a [`Receiver`] and for a +/// [`Root`] alike. +#[derive(Clone, Copy)] +enum Site<'a> { + Slot(usize), + Name(&'a str), +} + +/// What a chain turned out to be rooted at. +/// +/// Rhai draws this line in `search_namespace`, which hands back a `Target`: a +/// scope entry becomes a reference to write through, and a resolver's answer or +/// a module's constant becomes a read-only temporary (`eval/expr.rs:120-155`). +/// Which one a [`Root::Named`] is cannot be known until it is looked up. +enum RootAt<'a> { + /// A scope entry. The only root a chain writes back into, and only when it + /// is not a constant. + Place(Site<'a>), + /// A value with a name but no entry behind it — a resolver's answer, or a + /// module's constant. + Constant, + /// A value with no name either: `[1, 2].len()`, `f().x`. Nothing can be + /// assigned to one, because rhai's parser refuses it outright. + Temporary, +} + +/// A chain's root, looked up. +struct ChainRoot<'a> { + at: RootAt<'a>, + /// The value to walk. + /// + /// **Read-only if rhai's `Target` would have been**, which is not + /// decoration: cloning a `Dynamic` marks the copy read-write however the + /// original was (`types/dynamic.rs:822`), and the access mode is the only + /// thing standing between a `const` and a method that mutates it — + /// `exec_native_fn_call` refuses a non-pure function whose first argument + /// is read-only (`func/call.rs:405`). + value: Dynamic, + /// Where to blame a refusal: the variable for a name, the chain otherwise. + pos: Position, +} + +/// What one indexing step managed. +enum Indexed { + /// Taken through a reference: the value, and whether anything wrote. + Done(Dynamic, bool), + /// There was no reference to take. Carries the value back out, because the + /// caller cannot touch the container until this borrow has ended. + NoReference(Dynamic), +} + +/// What a chain's root is called, for the two errors that name it. +/// +/// `None` for a temporary, which has no name to give — and neither error can +/// reach one: nothing assigns through a temporary, and flattening it on the way +/// in means there is no cell left to contend for. +fn root_name<'p>(program: &'p Program, chain: &Chain) -> Option<&'p str> { + match chain.root { + Root::Local { name, .. } | Root::Named { name, .. } => program.name(name), + Root::Temporary => None, + } +} + +/// The op-assignment a chain ends with, resolved out of the pool. +fn chain_op<'p>( + program: &'p Program, + chain: &Chain, +) -> Result, Box> { + let Tail::Assign { op: Some(op) } = &chain.tail else { + return Ok(None); + }; + program + .assign_op(*op) + .map(Some) + .ok_or_else(|| malformed(format!("no op-assignment {op}"))) +} + +/// One `for` loop in progress. +/// +/// The count is here rather than in a local because rhai keeps it outside the +/// scope too, and checks it for overflow before writing it — a loop long +/// enough to wrap the counter is an error rather than a wrap +/// (`eval/stmt.rs:729`). +struct Iteration { + items: Box>, + /// The index of the item last handed out, starting one below the first. + count: INT, +} + +/// A scope entry as a place to write, seeing through a shared cell. +/// +/// Rhai reaches a variable through a `Target`, whose shared arm hands over the +/// cell's guard rather than the cell (`eval/target.rs:409-422`), so an +/// assignment lands where every closure holding that cell can see it. Writing +/// the slot itself would replace the cell and quietly sever them — the value +/// would be right and the aliasing dead. +/// +/// For an ordinary value `write_lock` is a downcast to itself, so the common +/// case pays nothing. Rhai's own for-loop does this with `.unwrap()` and +/// panics on a contended cell; a VM that promises errors instead of panics +/// reports `ErrorDataRace`, as `Target` does. +fn place<'a>( + entry: &'a mut Dynamic, + name: &str, + pos: Position, +) -> Result, Box> { + entry + .write_lock::() + .ok_or_else(|| Box::new(EvalAltResult::ErrorDataRace(name.to_string(), pos))) +} + +fn missing(name: &str, pos: Position) -> Box { + Box::new(EvalAltResult::ErrorVariableNotFound(name.to_string(), pos)) +} + +fn malformed(detail: String) -> Box { + Box::new(EvalAltResult::ErrorRuntime( + format!("malformed chunk: {detail}").into(), + Position::NONE, + )) +} + +/// Executes a [`Program`] against an `Engine`. +/// +/// Holds one `GlobalRuntimeState` and one `Caches` for its whole lifetime, so +/// the function-resolution cache survives across calls. That matters: the +/// reentrant helpers rhai exposes to native functions build a fresh +/// `Caches::new()` per call (`func/native.rs:519`), which would throw away +/// resolution work on every dispatch. +pub struct Vm<'e> { + engine: &'e Engine, + global: GlobalRuntimeState, + caches: Caches, + stack: Vec, + /// One entry per `for` loop currently running. + /// + /// Not on the operand stack, because an iterator is not a `Dynamic`. A + /// frame truncates this to what it found on entry, so a `return` or an + /// escaping error drops whatever its loops were holding without the + /// compiler emitting anything. + iterators: Vec, + /// One entry per `try` region currently armed or catching. Frame-floored + /// the same way the iterators are, so an error in a called function can + /// never find its caller's handler and jump into another chunk. + handlers: Vec, + /// The running data-size total of each literal currently being built, + /// innermost last. + /// + /// One entry per array or map literal under construction, so + /// `[a, [b, c], d]` keeps the inner total separate from the outer. + /// Truncated per frame, as the iterators are, so an error part way through + /// a literal leaves nothing behind. + sizes: Vec<(usize, usize, usize)>, + /// Where the scope goes back to if an error escapes the running frame. + /// + /// Set by [`Op::Checkpoint`](crate::bytecode::Op::Checkpoint) at each + /// top-level statement of a chunk, saved and restored per frame. See + /// [`Vm::execute`]. + unwind_floor: usize, + fault_pc: Option, +} + +/// A `try` region. +struct Handler { + target: usize, + catch_var: Option, + /// Where the three stacks were when the region was entered. An error can + /// be raised at any depth of all three, and the catch block has to begin + /// where the `try` did. + operands: usize, + scope_len: usize, + iters: usize, + /// Set once the catch block is running, holding the error it caught. That + /// is what a bare `throw;` in the catch block re-raises, and its presence + /// is what tells an escaping error it is leaving a catch rather than + /// entering one. + caught: Option>, +} + +impl<'e> Vm<'e> { + #[must_use] + pub fn new(engine: &'e Engine) -> Self { + Self { + engine, + global: engine.new_global_runtime_state(), + caches: Caches::new(), + stack: Vec::new(), + iterators: Vec::new(), + handlers: Vec::new(), + sizes: Vec::new(), + unwind_floor: 0, + fault_pc: None, + } + } + + /// A `Vm` for a call arriving from inside a native function. + /// + /// Reproduces what rhai does at every reentrant boundary + /// (`func/native.rs:516-519`, `types/fn_ptr.rs:451-454`): the caller's + /// runtime state is *cloned* rather than shared, and the resolution cache + /// starts empty. The clone is what carries the imported modules, the source + /// name and — the part that matters here — the function library holding the + /// wrappers, so a closure reached from a native can hand out a pointer of + /// its own. + /// + /// The empty `Caches` is the cost, and it is the one thing a `Vm` normally + /// exists to avoid. It cannot be helped: the outer `Vm` is borrowed by the + /// frame still running beneath this one. Rhai pays the same on its own + /// callbacks — but it also skips resolution entirely for a pointer that + /// carries its body, which is why a crossing measures 0.34x. See + /// [`callback`]. + /// + /// Operation counting has the same shape and the same reason: increments + /// inside the callback land on the clone and are lost when it drops, as + /// they are for any reentrant call rhai makes. + #[must_use] + pub fn reentrant(context: &'e NativeCallContext<'_>) -> Self { + Self { + engine: context.engine(), + global: context.global_runtime_state().clone(), + caches: Caches::new(), + stack: Vec::new(), + iterators: Vec::new(), + handlers: Vec::new(), + sizes: Vec::new(), + unwind_floor: 0, + fault_pc: None, + } + } + + /// Which instruction the last run failed at, if it failed. + /// + /// This is what a stripped program reports instead of a position. The host + /// that kept the table resolves it with `rhaigrain_pos::resolve`, so a + /// device can stay silent about where its source was and still produce a + /// diagnostic someone can act on. + /// + /// Cleared at the start of every run, so it always describes the most + /// recent one. + #[must_use] + pub fn fault_pc(&self) -> Option { + self.fault_pc + } + + /// Run a program, returning its value. + /// + /// Mirrors `Engine::eval_ast_with_scope_raw`: the program's function + /// library, module resolver and source name are installed for the duration + /// and restored afterwards, so a `Vm` reused across programs does not leak + /// one program's definitions into the next. + /// Call one compiled function by name, with arguments already evaluated. + /// + /// The entry point a native needs. `Op::Call` reaches a chunk through the + /// name *index* it shares with the call site, which a caller from outside + /// does not have — a `FnPtr` carries a string, and so does rhai when it + /// dispatches. This is the same call by the other key. + /// + /// `level` is the caller's call depth, so `max_call_levels` still counts + /// across a boundary that leaves this VM and comes back. Left unthreaded, + /// a closure calling itself through `map` would recurse until the stack + /// went rather than until the limit did. + /// + /// # Errors + /// + /// `ErrorFunctionNotFound` if no compiled function has that name and + /// arity, and whatever the function itself raises otherwise. + pub fn call_function( + &mut self, + program: &Program, + name: &str, + args: Vec, + level: usize, + pos: Position, + ) -> VmResult { + let Some(function) = program.function_named(name, args.len()) else { + return Err(Box::new(EvalAltResult::ErrorFunctionNotFound( + format!("{name} ({} args)", args.len()), + pos, + ))); + }; + let (params, chunk) = (function.params.clone(), function.chunk); + + // `call_compiled` takes its arguments off the operand stack, where a + // compiled call site would already have put them. + let first = self.stack.len(); + self.stack.extend(args); + + let restore = mem::replace(&mut self.global.level, level); + let result = self.call_compiled(program, name, ¶ms, chunk, first, pos); + self.global.level = restore; + + self.stack.truncate(first); + result + } + + pub fn run(&mut self, program: &Program, scope: &mut Scope) -> VmResult { + self.run_with(program, scope, None) + } + + /// Run a program that hands function pointers to native functions. + /// + /// The same run, plus one native wrapper per compiled function registered + /// for its duration, so a pointer this program creates resolves when rhai + /// dispatches it — `let a = [1, 2]; a.map(|x| x * 2)` is `map` calling us + /// back, and `map` looks the pointer up its own way. See [`callback`]. + /// + /// Only worth the owned program when [`Program::makes_fn_pointers`] says a + /// pointer can escape; [`run`](Self::run) is otherwise identical and copies + /// nothing. A program that needs this and does not get it still runs — the + /// pointer simply fails to resolve, as `ErrorFunctionNotFound`, at the + /// point the native tries to call it. + /// + /// Read [`callback`] before relying on it: a crossing is slower than the + /// walker, and a *capturing* closure handed to a native that binds `this` + /// arrives with its arguments rotated. + pub fn run_with_callbacks(&mut self, program: &SharedProgram, scope: &mut Scope) -> VmResult { + let wrappers = + (!program.functions().is_empty()).then(|| callback::wrappers(program).into()); + self.run_with(program, scope, wrappers) + } + + fn run_with( + &mut self, + program: &Program, + scope: &mut Scope, + wrappers: Option, + ) -> VmResult { + let orig_source = mem::replace(&mut self.global.source, program.source().cloned()); + let orig_lib_len = self.global.lib.len(); + if let Some(lib) = program.lib() { + self.global.lib.push(lib.clone()); + } + // Last, so the search — which runs in reverse — reaches a compiled + // function before whatever the compiler left rhai to interpret. + if let Some(wrappers) = wrappers { + self.global.lib.push(wrappers); + } + #[cfg(not(feature = "no_module"))] + let orig_resolver = mem::replace( + &mut self.global.embedded_module_resolver, + program.resolver().cloned(), + ); + + self.fault_pc = None; + let mut pc = program.main().entry() as usize; + // Slots are indices into the caller's scope, so a caller that arrives + // with variables already in it shifts every one of them. + let base = scope.len(); + let result = self.execute(program, scope, *program.main(), base, &mut pc); + if result.is_err() { + self.fault_pc = Some(pc); + } + + #[cfg(not(feature = "no_module"))] + { + self.global.embedded_module_resolver = orig_resolver; + } + self.global.lib.truncate(orig_lib_len); + self.global.source = orig_source; + + // Rhai unwinds `return` and `exit` as errors rather than returning + // them; `eval_global_statements` is where they turn back into values, + // and anything entering a program from outside has to do the same. + result.or_else(|err| match *err { + EvalAltResult::Return(out, ..) | EvalAltResult::Exit(out, ..) => Ok(out), + _ => Err(err), + }) + } + + fn pop(&mut self) -> Result> { + self.stack + .pop() + .ok_or_else(|| malformed("operand stack underflow".to_string())) + } + + /// `reached` tracks the instruction being executed, so a failure can be + /// attributed to one. It is what a stripped program reports in place of a + /// position. + /// Call a chunk this compiler produced, reproducing `call_script_fn` + /// (`func/script.rs:24`) step for step. + /// + /// The parts that are not obvious, and that the differential corpus is + /// what proves: arguments are *taken* out of the caller's stack slots + /// rather than cloned, the depth check happens after the level is + /// incremented, and only errors that are neither a `return` nor a system + /// exception get wrapped in `ErrorInFunctionCall`. + /// Walk `a.b[i].c`, reading it or assigning to it. + /// + /// The reason this is one instruction and a recursion rather than a + /// sequence: every level holds a `&mut` into the level above, exactly as + /// rhai does (`eval/chaining.rs:659`). For a map or an array that borrow is + /// the whole story — the mutation lands in the container and no write-back + /// is needed. Doing it on the operand stack instead would mutate a copy. + /// + /// Write-back is only for the levels where a borrow was not possible: + /// a getter on a host type hands back a value, and rhai calls the setter + /// afterwards if the sub-chain was a method call. `changed` reproduces + /// that, and it is deliberately coarse in the same way rhai's is — rhai's + /// flag is `func.is_method()`, "does the resolved function take its + /// receiver by reference", not "did it actually write". + #[inline(never)] + fn run_chain( + &mut self, + program: &Program, + chain: &Chain, + scope: &mut Scope, + base: usize, + pos: Position, + ) -> VmResult { + // Step operands were pushed first, then the root if it is one that has + // to be evaluated, then the value being assigned. + let operands_at = self + .stack + .len() + .checked_sub(chain.consumes()) + .ok_or_else(|| malformed("chain with too few operands".to_string()))?; + + let ChainRoot { + at, + value: mut root, + pos: root_pos, + } = self.chain_root(program, chain, scope, base, operands_at, pos)?; + + // Read-only is what refuses an assignment, not the absence of a place: + // a module's constant is neither, so rhai assigns into the copy and + // discards it. Only a `const` and a resolver's answer are refused, and + // both are read-only for that reason. + // + // A temporary is separate again — rhai's parser refuses `f().x = 1` + // outright, so one reaches here only from a chunk this compiler did + // not build. + let value = match (chain.assigns(), &at) { + (false, _) => None, + (true, RootAt::Temporary) => { + return Err(malformed("chain assigns through a temporary root".into())) + } + (true, _) if root.is_read_only() => { + let name = root_name(program, chain) + .ok_or_else(|| malformed("no chain root name".to_string()))?; + return Err(Box::new(EvalAltResult::ErrorAssignmentToConstant( + name.to_string(), + root_pos, + ))); + } + // Rhai flattens the right-hand side before assigning, so a shared + // cell is copied out rather than aliased in. + (true, _) => Some(self.stack[self.stack.len() - 1].clone().flatten()), + }; + + let mut operands: Vec = + self.stack[operands_at..operands_at + chain.operands as usize].to_vec(); + + // A shared cell cannot be walked directly. `get_indexed_mut` refuses + // one outright — `unreachable!("cannot handle shared values")`, + // `eval/chaining.rs:461` — because rhai always reaches a root through + // a `Target`, whose shared arm hands over the guard rather than the + // cell. Walking the cell would take the host down, so this is a + // panic-safety fix and not only a correctness one. + // + // Nothing is written back for a shared root: cloning a shared + // `Dynamic` clones the `Rc`, so a mutation through the guard already + // landed in the cell every other holder can see. + let shared = is_shared!(root); + let result = if shared { + let mut guard = root.write_lock::().ok_or_else(|| { + let name = root_name(program, chain).unwrap_or_default(); + Box::new(EvalAltResult::ErrorDataRace(name.to_string(), pos)) + })?; + self.walk_chain( + program, + chain, + &chain.steps, + &mut guard, + &mut operands, + value, + pos, + ) + } else { + self.walk_chain( + program, + chain, + &chain.steps, + &mut root, + &mut operands, + value, + pos, + ) + }; + + // A place is the one root left that writes back — the entry cannot be + // held across the walk without borrowing the scope for its whole + // duration, so the walk gets a copy and this puts it back. + // + // Not a constant, which could not have been changed anyway: the walk + // was handed a read-only value, so anything that would have mutated it + // refused rather than mutating the copy. + if chain.mutates() && !shared && result.is_ok() && !root.is_read_only() { + match at { + RootAt::Place(Site::Slot(index)) => *scope.get_mut_by_index(index) = root, + RootAt::Place(Site::Name(name)) => { + let entry = scope + .get_mut(name) + .ok_or_else(|| malformed(format!("`{name}` stopped being writable")))?; + *entry = root; + } + RootAt::Constant | RootAt::Temporary => {} + } + } + + let (out, _) = result?; + self.stack.truncate(operands_at); + Ok(out) + } + + /// Find what a chain is rooted at, resolving a name if that is what it is. + /// + /// The search is `load_named`'s and the order is observable: a resolver + /// registered with `Engine::on_var` sees the name before the scope does, + /// and a name in no scope is looked for among the global modules before it + /// is reported missing. It runs exactly once — a chain is one instruction, + /// so unlike [`Op::CallRef`] there is nothing to resolve twice. + /// + /// `ErrorVariableNotFound` is reported against the *variable*, which is why + /// [`Root::Named`] carries a position of its own. + fn chain_root<'p>( + &mut self, + program: &'p Program, + chain: &Chain, + scope: &mut Scope, + base: usize, + operands_at: usize, + pos: Position, + ) -> Result, Box> { + // The walk gets a copy of the entry, and cloning a `Dynamic` marks the + // copy read-write however the original was — so a constant has to be + // told it came from one. See [`ChainRoot::value`]. + let walkable = |value: &Dynamic| { + if value.is_read_only() { + value.clone().into_read_only() + } else { + value.clone() + } + }; + + match chain.root { + Root::Local { slot, .. } => { + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!( + "chain root slot {index} is out of scope" + ))); + } + Ok(ChainRoot { + at: RootAt::Place(Site::Slot(index)), + value: walkable(scope.get_mut_by_index(index)), + pos, + }) + } + + // A name has a position of its own, and it wins: the lookup below + // can fail, and rhai blames the variable rather than the chain. + Root::Named { name, pos: var_pos } => { + let name = program + .name(name) + .ok_or_else(|| malformed(format!("no name {name}")))?; + + // A resolver hands back a value rather than a place, which is + // what makes writing through it an error. + if let Some(value) = self.resolve_var(name, scope, var_pos)? { + return Ok(ChainRoot { + at: RootAt::Constant, + value: value.into_read_only(), + pos: var_pos, + }); + } + if let Some(value) = scope.get(name) { + return Ok(ChainRoot { + at: RootAt::Place(Site::Name(name)), + value: walkable(value), + pos: var_pos, + }); + } + // A constant a host published with `Module::set_var`. Not + // marked read-only, because rhai does not mark it either + // (`eval/expr.rs:151` against `:122`) — so a chain assigns + // into the copy and discards it, where writing to the name + // directly is refused. + self.engine + .global_modules + .iter() + .find_map(|module| module.get_var(name)) + .map(|value| ChainRoot { + at: RootAt::Constant, + value, + pos: var_pos, + }) + .ok_or_else(|| missing(name, var_pos)) + } + + Root::Temporary => Ok(ChainRoot { + at: RootAt::Temporary, + value: self.stack[operands_at + chain.operands as usize] + .clone() + .flatten(), + pos, + }), + } + } + + /// One level of the walk. Returns the value and whether anything below may + /// have written. + #[allow(clippy::too_many_arguments)] + fn walk_chain( + &mut self, + program: &Program, + chain: &Chain, + steps: &[Step], + target: &mut Dynamic, + operands: &mut [Dynamic], + value: Option, + pos: Position, + ) -> Result<(Dynamic, bool), Box> { + let Some((step, rest)) = steps.split_first() else { + // The end of the chain, reached with nothing to do: a bare `a` is + // not a chain, so this only happens for an empty step list. + return Ok((target.clone(), false)); + }; + let last = rest.is_empty(); + + match step { + Step::Index { + operand, + pos: idx_pos, + bracket, + } => { + let idx = operands + .get_mut(*operand as usize) + .ok_or_else(|| malformed("chain index operand missing".to_string()))?; + let mut idx = idx.clone(); + // Rhai reports an out-of-bounds index against the index and a + // value that cannot be indexed at all against this step's `[`. + // Both belong to the step, and neither is the chain's. + let idx_pos = *idx_pos; + let bracket = *bracket; + + // Split out so the borrow `get_indexed_mut` takes of `target` + // ends when it returns: the fallback below needs `target` + // again, and a `Target` in scope would still be holding it. + match self.index_by_reference( + program, chain, rest, target, &mut idx, idx_pos, operands, value, last, + bracket, pos, + )? { + Indexed::Done(out, changed) => Ok((out, changed)), + Indexed::NoReference(value) => { + self.assign_through_indexer( + program, chain, target, &mut idx, value, bracket, + )?; + Ok((Dynamic::UNIT, true)) + } + } + } + + Step::Property { + name, + getter, + setter, + pos: step_pos, + } => self.walk_property( + program, chain, rest, target, operands, value, pos, *step_pos, *name, *getter, + *setter, + ), + + Step::Method { + name, + argc, + operand, + pos: step_pos, + } => { + let step_pos = *step_pos; + let name = program + .name(*name) + .ok_or_else(|| malformed(format!("no name {name}")))?; + let first = *operand as usize; + let argc = *argc as usize; + if first + argc > operands.len() { + return Err(malformed("chain method arguments missing".to_string())); + } + + let mut args: Vec = operands[first..first + argc].to_vec(); + let out = { + let mut call_args: Vec<&mut Dynamic> = core::iter::once(&mut *target) + .chain(args.iter_mut()) + .collect(); + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, true, true, &mut call_args) + .map_err(|err| dispatch_failure(err, step_pos))? + }; + + if last { + match value { + // `a.f() = x` is not something rhai parses. + Some(_) => Err(malformed("assignment to a method call".to_string())), + None => Ok((out, true)), + } + } else { + let mut inner = out; + let (out, _) = + self.walk_chain(program, chain, rest, &mut inner, operands, value, pos)?; + // Whatever the sub-chain did, it did to the method's + // return value, which nothing owns. + Ok((out, true)) + } + } + } + } + + /// One `[i]` step, taken through a reference into the container. + /// + /// Returns [`Indexed::NoReference`] when there is no reference to be had — + /// a custom indexer being assigned through — handing the value back so the + /// caller can take the long way round once this borrow has ended. + #[allow(clippy::too_many_arguments)] + fn index_by_reference( + &mut self, + program: &Program, + chain: &Chain, + rest: &[Step], + target: &mut Dynamic, + idx: &mut Dynamic, + idx_pos: Position, + operands: &mut [Dynamic], + value: Option, + last: bool, + bracket: Position, + pos: Position, + ) -> Result> { + let assigning = last && value.is_some(); + let mut detached = Scope::new(); + + let mut item = match self.engine.get_indexed_mut( + &mut self.global, + &mut self.caches, + &mut detached, + None, + target, + idx, + idx_pos, + bracket, + // Auto-vivify a missing map key only when writing, as rhai does + // for the assignment case (`eval/chaining.rs:791`). + assigning, + // And do not reach for a custom indexer when writing: a value it + // handed back could not be assigned through. Rhai asks the same + // way and takes the error as its signal. + !assigning, + ) { + Ok(item) => item, + Err(err) if assigning && matches!(*err, EvalAltResult::ErrorIndexingType(..)) => { + return Ok(Indexed::NoReference(value.expect("assigning"))); + } + Err(err) => return Err(err), + }; + + // A read changes nothing, so it consumes the target and there is + // nothing to put back. + if last && value.is_none() { + return Ok(Indexed::Done(item.take_or_clone(), false)); + } + + let temp = item.is_temp_value(); + let (out, changed) = if last { + let value = value.expect("checked above"); + self.store( + program, + chain_op(program, chain)?, + item.as_mut(), + value, + pos, + )?; + (Dynamic::UNIT, true) + } else { + // Straight through the borrow: for an array, a map or a blob this + // *is* the container's element, so a mutation below lands where + // rhai's would. + self.walk_chain(program, chain, rest, item.as_mut(), operands, value, pos)? + }; + + // Bit-fields, string characters and blob bytes cannot be pointed at + // directly, so `Target` carries a copy and this is what puts it back + // (`eval/target.rs:282`). + item.propagate_changed_value(pos)?; + + if temp && changed { + // The element was a temporary — a custom indexer's — so the setter + // is the only way back (`eval/chaining.rs:744`). + let mut updated = item.take_or_clone(); + let mut index = idx.clone(); + self.call_indexer_set(target, &mut index, &mut updated, bracket)?; + } + + Ok(Indexed::Done(out, changed)) + } + + /// Assign through a custom indexer, which cannot hand out a reference. + /// + /// An op-assignment has to read the current value back through the getter + /// first, and rhai *ignores* a getter that fails here — a write-only + /// indexer takes the new value as-is (`eval/chaining.rs:812`). + fn assign_through_indexer( + &mut self, + program: &Program, + chain: &Chain, + target: &mut Dynamic, + index: &mut Dynamic, + value: Dynamic, + pos: Position, + ) -> Result<(), Box> { + let mut new_val = value; + + if matches!(chain.tail, Tail::Assign { op: Some(_) }) { + let mut probe = index.clone(); + if let Ok(mut current) = self.call_indexer(FN_IDX_GET, target, &mut probe, pos) { + self.store( + program, + chain_op(program, chain)?, + &mut current, + new_val, + pos, + )?; + new_val = current; + } + } + + self.call_indexer_set(target, index, &mut new_val, pos) + } + + /// Call the index getter, which unlike the setter is allowed to fail. + fn call_indexer( + &mut self, + name: &str, + target: &mut Dynamic, + index: &mut Dynamic, + pos: Position, + ) -> VmResult { + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, true, false, &mut [target, index]) + .map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos); + } + err + }) + } + + /// Put an element back into a container that had no reference to give. + /// + /// A custom indexer returns a value, so a mutation below it landed in a + /// temporary; this is the replay rhai does at `eval/chaining.rs:744`, + /// including swallowing "this type cannot be indexed" the way it does. + fn call_indexer_set( + &mut self, + target: &mut Dynamic, + index: &mut Dynamic, + value: &mut Dynamic, + pos: Position, + ) -> Result<(), Box> { + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + + match context.call_fn_raw(FN_IDX_SET, true, false, &mut [target, index, value]) { + Ok(_) => Ok(()), + Err(err) if matches!(*err, EvalAltResult::ErrorIndexingType(..)) => Ok(()), + Err(mut err) => { + if err.position().is_none() { + err.set_position(pos); + } + Err(err) + } + } + } + + /// `.name`, which is a key on a map and a getter call on anything else. + /// + /// The distinction is rhai's and it is made at runtime, not at parse time + /// (`eval/chaining.rs:898`). It matters for more than speed: a map hands + /// back a reference, so a mutation below lands in the map, while a getter + /// hands back a value that has to be given to the setter afterwards. + #[allow(clippy::too_many_arguments)] + fn walk_property( + &mut self, + program: &Program, + chain: &Chain, + rest: &[Step], + target: &mut Dynamic, + operands: &mut [Dynamic], + value: Option, + pos: Position, + // The property's own position, which is where rhai blames a getter or + // setter that does not exist (`eval/chaining.rs:1039`). `pos` is the + // chain's, and stays that for everything else. + step_pos: Position, + name: u32, + getter: u32, + setter: u32, + ) -> Result<(Dynamic, bool), Box> { + let last = rest.is_empty(); + let key = program + .name(name) + .ok_or_else(|| malformed(format!("no name {name}")))?; + + if target.is_map() { + let mut map = target + .write_lock::() + .ok_or_else(|| malformed("a map that is not a map".to_string()))?; + + // Only a write creates a key. Rhai passes `add_if_not_found` for + // an assignment (`eval/chaining.rs:930`) and withholds it for a + // read (`:959`) and for a step on the way through (`:1086`), so + // reading `m.absent` must leave `m` alone — otherwise a closure + // holding the map sees a key nobody wrote. + if last { + if let Some(value) = value { + let entry = map.entry(key.into()).or_insert(Dynamic::UNIT); + self.store(program, chain_op(program, chain)?, entry, value, pos)?; + return Ok((Dynamic::UNIT, true)); + } + return match map.get(key) { + Some(entry) => Ok((entry.clone(), false)), + None => self.absent_key(key, step_pos).map(|unit| (unit, false)), + }; + } + + return match map.get_mut(key) { + Some(entry) => self.walk_chain(program, chain, rest, entry, operands, value, pos), + // Rhai walks on into a detached unit, so whatever the rest of + // the chain does to it is discarded (`eval/chaining.rs:211`). + None => { + let mut absent = self.absent_key(key, step_pos)?; + drop(map); + self.walk_chain(program, chain, rest, &mut absent, operands, value, pos) + } + }; + } + + // A host type: getter in, setter out. + let call = |vm: &mut Self, fn_name: u32, args: &mut [&mut Dynamic]| -> VmResult { + let fn_name = program + .name(fn_name) + .ok_or_else(|| malformed(format!("no name {fn_name}")))?; + let mut detached = Scope::new(); + let mut context = EvalContext::new( + vm.engine, + &mut vm.global, + &mut vm.caches, + &mut detached, + None, + ); + context + .call_fn_raw(fn_name, true, true, args) + .map_err(|err| positioned(err, step_pos)) + }; + + if last { + if let Some(value) = value { + // `x.p += 1` has to read `p` back through the getter before it + // can add to it — the setter takes a finished value. + let mut new_val = if matches!(chain.tail, Tail::Assign { op: Some(_) }) { + let mut current = call(self, getter, &mut [target])?; + self.store(program, chain_op(program, chain)?, &mut current, value, pos)?; + current + } else { + value + }; + // A setter's return value is thrown away, as in rhai. + let _ = call(self, setter, &mut [target, &mut new_val])?; + return Ok((Dynamic::UNIT, true)); + } + let out = call(self, getter, &mut [target])?; + return Ok((out, false)); + } + + // A getter returns a value, so the rest of the chain works on a + // temporary. Rhai puts it back through the setter when the sub-chain + // was a method call, and skips the setter otherwise. + let mut temp = call(self, getter, &mut [target])?; + let (out, changed) = + self.walk_chain(program, chain, rest, &mut temp, operands, value, pos)?; + if changed { + let _ = call(self, setter, &mut [target, &mut temp])?; + } + Ok((out, changed)) + } + + /// Store into a slot the walk arrived at, through an operator if there is + /// one. + /// + /// Same resolution order as a plain local assignment, and for the same + /// reason: `x += y` is not `x = x + y` unless nothing implements `+=`. + /// The built-in op-assignment for these operands, if rhai has one. + /// + /// Inlined deliberately: `x += 1` in a loop is entirely this, and routing + /// it through the out-of-line resolution below measured 10% on the + /// tight-loop benchmark. Inlining the *whole* of `store` instead costs + /// more than it saves — it took `branch heavy` from 1.59x to 1.41x — so + /// the split is where the two paths part. + #[inline] + fn store_builtin( + &mut self, + op: &AssignOp, + target: &mut Dynamic, + rhs: &mut Dynamic, + pos: impl Fn() -> Position, + ) -> Option>> { + if !self.engine.fast_operators() { + return None; + } + let (func, need_context) = get_builtin_op_assignment_fn(&op.op_assign, target, rhs)?; + let context = + need_context.then(|| native_context(self.engine, "", None, &self.global, pos())); + Some( + func(context, &mut [target, rhs]) + .map(|_| ()) + .map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos()); + } + err + }), + ) + } + + fn store( + &mut self, + program: &Program, + op: Option<&AssignOp>, + target: &mut Dynamic, + mut rhs: Dynamic, + pos: Position, + ) -> Result<(), Box> { + let Some(op) = op else { + *target = rhs; + return Ok(()); + }; + + if let Some(done) = self.store_builtin(op, target, &mut rhs, || pos) { + return done; + } + + let op_assign_name = program + .name(op.op_assign_name) + .ok_or_else(|| malformed("no op-assign name".to_string()))?; + let op_name = program + .name(op.op_name) + .ok_or_else(|| malformed("no operator name".to_string()))?; + + // The real scope may be borrowed by the target, and dispatch does not + // read it anyway — operators resolve against the engine. + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + + match context.call_fn_raw(op_assign_name, true, false, &mut [target, &mut rhs]) { + Ok(_) => Ok(()), + Err(err) + if matches!(&*err, + EvalAltResult::ErrorFunctionNotFound(name, ..) + if name.starts_with(op_assign_name)) => + { + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + let value = context + .call_fn_raw(op_name, true, false, &mut [&mut *target, &mut rhs]) + .map_err(|err| positioned(err, pos))?; + *target = value; + Ok(()) + } + Err(err) => Err(positioned(err, pos)), + } + } + + /// Read a variable no slot names, the way rhai's `search_scope_only` does + /// (`eval/expr.rs:107-155`). + /// + /// Three places in a fixed order, and the order is observable: a resolver + /// registered with `Engine::on_var` sees the name before the scope does, + /// and a name in no scope is looked for among the global modules before it + /// is reported missing. + /// + /// `flatten` is what the two reads differ by, and only for a scope entry: + /// a value position wants what a shared cell contains, and a capture wants + /// the cell. The other two places can only ever produce a value. + /// + /// Kept out of the dispatch loop for the reason [`Vm::call_compiled`] is. + #[inline(never)] + fn load_named( + &mut self, + name: &str, + scope: &mut Scope, + flatten: bool, + pos: Position, + ) -> VmResult { + // A resolver hands back a value, not a place, so it is read-only — + // which is what makes assigning to one an error. + if let Some(value) = self.resolve_var(name, scope, pos)? { + return Ok(value.into_read_only()); + } + + if let Some(value) = scope.get(name) { + return Ok(if flatten { + value.flatten_clone() + } else { + value.clone() + }); + } + + // A constant a host published with `Module::set_var`. + if let Some(value) = self + .engine + .global_modules + .iter() + .find_map(|module| module.get_var(name)) + { + return Ok(value); + } + + Err(missing(name, pos)) + } + + /// Ask the resolver a host registered with `Engine::on_var`, if there is + /// one. + /// + /// `Ok(None)` covers both "no resolver" and "the resolver declined", which + /// are the same thing to every caller. + fn resolve_var( + &mut self, + name: &str, + scope: &mut Scope, + pos: Position, + ) -> Result, Box> { + // Copied out so the borrow is of the engine rather than of `self`, + // which the context below needs mutably. + let engine = self.engine; + let Some(resolver) = &engine.resolve_var else { + return Ok(None); + }; + + let before = scope.len(); + let context = EvalContext::new(engine, &mut self.global, &mut self.caches, scope, None); + // Index zero: rhai passes the slot its parser resolved, and a name + // that reached here had none. + let resolved = resolver(name, 0, context); + + // A resolver that pushed onto the scope has moved every entry a + // parse-time index named, so rhai stops trusting those from here on. + // Nothing this compiler emits depends on them — its slots are counted + // from a base taken before the run — but a fragment's do. + if scope.len() != before { + self.global.always_search_scope = true; + } + + resolved.map_err(|err| { + if err.position().is_none() { + return reposition(err, pos); + } + err + }) + } + + /// Assign to a variable no slot names. + /// + /// Rhai reaches the target through the same search and then refuses + /// anything that is not a reference it can write through: a value the + /// resolver produced, a module's constant, a `const` entry + /// (`eval/stmt.rs:330-344` and `eval/stmt.rs:118-122`). All three are + /// `ErrorAssignmentToConstant`, so the distinction never reaches a script. + #[inline(never)] + fn assign_named( + &mut self, + program: &Program, + op: Option<&AssignOp>, + name: &str, + rhs: Dynamic, + scope: &mut Scope, + pos: Position, + ) -> Result<(), Box> { + let constant = || { + Box::new(EvalAltResult::ErrorAssignmentToConstant( + name.to_string(), + pos, + )) + }; + + if self.resolve_var(name, scope, pos)?.is_some() { + return Err(constant()); + } + + match scope.is_constant(name) { + Some(true) => return Err(constant()), + Some(false) => {} + // Not a variable at all. A module's is a value rather than a + // place, so writing to one is the same refusal as writing to a + // `const`. + None => { + return Err( + if self + .engine + .global_modules + .iter() + .any(|module| module.get_var(name).is_some()) + { + constant() + } else { + missing(name, pos) + }, + ) + } + } + + let entry = scope + .get_mut(name) + .ok_or_else(|| malformed(format!("`{name}` is in scope but not writable")))?; + let mut target = place(entry, name, pos)?; + + self.store(program, op, &mut target, rhs, pos) + } + + /// Call a function pointer, preferring a chunk we compiled. + /// + /// The pointer sits under its arguments. Rhai's own dispatch would work + /// for all of this, but it cannot reach our chunks — the compiled function + /// table is keyed on names from the pool, and a pointer carries a string — + /// so the name is matched against it first and only the miss goes to + /// `call_raw`. + #[inline(never)] + fn call_fn_ptr( + &mut self, + program: &Program, + argc: usize, + method: bool, + pos: Position, + ) -> VmResult { + let base = self + .stack + .len() + .checked_sub(argc + 1) + .ok_or_else(|| malformed("function pointer call is missing its target".into()))?; + let mut at = base; + + // In method position a target that is not a pointer means the *first + // argument* is one and the target is the receiver — `obj.call(f, ..)` + // is how a closure is called against a `this`. Rhai reports the + // mismatch against that argument, not against the target, which is why + // the position moves with it. + let mut this = None; + if method && !self.stack[at].is::() { + this = Some(self.stack[at].clone()); + at += 1; + if at >= self.stack.len() { + return Err(self.mismatch::(self.stack[at - 1].type_name(), pos)); + } + } + + let pointer = self.stack[at] + .clone() + .try_cast::() + .ok_or_else(|| self.mismatch::(self.stack[at].type_name(), pos))?; + + let taken = self.stack.len() - at - 1; + let curried = pointer.curry().len(); + let function = (this.is_none()) + .then(|| program.function_named(pointer.fn_name(), curried + taken)) + .flatten() + .map(|f| (f.params.clone(), f.chunk)); + + if let Some((params, chunk)) = function { + // Curried arguments go in front of the call's own, which is what + // currying means and where the callee's parameters expect them. + let first = at + 1; + self.stack + .splice(first..first, pointer.curry().iter().cloned()); + let value = + self.call_compiled(program, pointer.fn_name(), ¶ms, chunk, first, pos)?; + self.stack.truncate(base); + return Ok(value); + } + + // Anything else is rhai's: a native function, a name registered + // elsewhere, or a pointer it built itself. + let mut args: Vec = self.stack.drain(at + 1..).collect(); + let context = native_context(self.engine, pointer.fn_name(), None, &self.global, pos); + let value = pointer + .call_raw(&context, this.as_mut(), &mut args) + .map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos); + } + err + })?; + self.stack.truncate(base); + Ok(value) + } + + /// Concatenate the segments of an interpolated string, reproducing + /// `eval/expr.rs:280-304`. + /// + /// Every step of it is load-bearing. A **string** segment is written + /// straight out and never reaches dispatch, so a host's `to_string` for + /// strings is not consulted here even though `+` would consult it. + /// Anything else goes through rhai's own rendering, which calls **native** + /// functions only — a script `fn to_string` is invisible to it — and + /// substitutes the mapped type name when the call returns a non-string. + /// The size limit is checked after every segment against the running + /// total, not once at the end. + #[inline(never)] + fn append_segment( + &mut self, + segment: Dynamic, + pos: Position, + ) -> Result<(), Box> { + use core::fmt::Write; + + let mut item = segment.flatten(); + let mut rendered = None; + + // A string is written straight out and never reaches dispatch, so a + // host's `to_string` for strings is not consulted here even though `+` + // would consult it. + if !item.is_string() { + let context = native_context(self.engine, FUNC_TO_STRING, None, &self.global, pos); + rendered = Some(print_with_func(FUNC_TO_STRING, &context, &mut item)); + } + + let mut buffer = self + .stack + .last_mut() + .and_then(|value| value.write_lock::()) + .ok_or_else(|| malformed("interpolation lost its buffer".into()))?; + + // `make_mut` is in place while the buffer is uniquely held, which on + // the operand stack it is — so this is one growing allocation rather + // than one per segment. + match rendered { + Some(text) => write!(buffer.make_mut(), "{text}"), + None => write!(buffer.make_mut(), "{item}"), + } + .expect("writing to a string cannot fail"); + let len = buffer.len(); + drop(buffer); + + // After every segment, against the running total — a script must not + // be able to build a string past `max_string_size` and hand it over + // whole. + self.engine.throw_on_size((0, 0, len)).map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos); + } + err + }) + } + + /// Start iterating a value, the way rhai's `for` does + /// (`eval/stmt.rs:680-703`). + /// + /// Three places are searched by `TypeId`, in order, and the order is + /// rhai's: the modules in the global namespace, then the imports, then the + /// statically registered sub-modules. Nothing matching is `ErrorFor`. + /// + /// The iterable is flattened first — so iterating a captured array walks a + /// snapshot rather than the shared cell — and is consumed by value, which + /// is why the iterator is built once and held for the life of the loop. + #[inline(never)] + fn iter_init(&mut self, iterable: Dynamic, pos: Position) -> Result<(), Box> { + let iterable = iterable.flatten(); + let type_id = iterable.type_id(); + + let func = self + .engine + .global_modules + .iter() + .find_map(|module| module.get_iter(type_id)); + + // Imported and sub-modules can register iterators too, but neither + // exists to be searched under `no_module`. + #[cfg(not(feature = "no_module"))] + let func = func.or_else(|| self.global.get_iter(type_id)).or_else(|| { + self.engine + .global_sub_modules + .values() + .find_map(|module| module.get_qualified_iter(type_id)) + }); + + let func = func.ok_or_else(|| Box::new(EvalAltResult::ErrorFor(pos)))?; + + self.iterators.push(Iteration { + items: func(iterable), + count: -1, + }); + Ok(()) + } + + /// Call `name` with `argc` arguments sitting contiguously from `first` up. + /// + /// A function this compiler lowered is called directly, with no hash and no + /// module walk: the call site's name index and the function's come from the + /// same pool, so equal names have equal indices. Everything else goes to + /// rhai's dispatch, and resolves exactly as it would in the walker. + fn call_stacked( + &mut self, + program: &Program, + name_index: u32, + argc: usize, + first: usize, + pos: Position, + ) -> VmResult { + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + + if let Some(function) = program.function(name_index, argc) { + return self.call_compiled(program, name, &function.params, function.chunk, first, pos); + } + + // Arguments are already contiguous at the top of the operand stack, + // which is exactly the shape rhai's ABI wants (`func/call.rs:36`). It + // consumes them, replacing each with unit, so the caller truncates + // afterwards rather than reusing them. + let mut args: Vec<&mut Dynamic> = self.stack[first..].iter_mut().collect(); + + // A scope of the callee's own, because the scope an `EvalContext` + // carries is the one a *script* function's body runs in + // (`func/call.rs:639`), and rhai passes `None` there (`:1476`). + // Handing over this frame's would let such a body read the caller's + // locals — reachable, because the functions this compiler skips are + // exactly the ones rhai can still find in `global.lib`. + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, false, false, &mut args) + .map_err(|err| dispatch_failure(err, pos)) + } + + /// The same call, with a variable as its first argument and rhai's + /// method-call rewrite applied to it (`func/call.rs:1434-1460`). + /// + /// The other arguments are already on the operand stack and were evaluated + /// before the receiver was reached, which is the order rhai uses and is + /// observable whenever one of them writes to the receiver. + #[inline(never)] + #[allow(clippy::too_many_arguments)] + fn call_by_reference( + &mut self, + program: &Program, + name_index: u32, + argc: usize, + receiver: Receiver, + scope: &mut Scope, + base: usize, + pos: Position, + ) -> VmResult { + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + + // Every argument count here includes the receiver, so zero of them + // names no receiver at all and the instruction is nonsense. Only an + // artifact can say it; the compiler emits one of these for a call that + // has a first argument. + if argc == 0 { + return Err(malformed( + "a call by reference with no receiver".to_string(), + )); + } + + // A named receiver's value is already argument zero — [`Op::LoadNamed`] + // put it there. A local's is not on the stack at all. + let (at, on_stack) = match receiver { + Receiver::Local(slot) => { + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + (Site::Slot(index), argc - 1) + } + Receiver::Named(var) => { + let name = program + .name(var) + .ok_or_else(|| malformed(format!("no name {var}")))?; + (Site::Name(name), argc) + } + }; + let first = self + .stack + .len() + .checked_sub(on_stack) + .ok_or_else(|| malformed("call with too few arguments".to_string()))?; + + let place = match at { + Site::Slot(index) => Some(scope.get_mut_by_index(index)), + // A resolver's answer shadows the scope, and `load_named` marks one + // read-only precisely because it is a value and not a place. Asking + // the resolver again to find that out would run it twice, which a + // host can see. + Site::Name(..) if self.stack[first].is_read_only() => None, + Site::Name(name) => scope.get_mut(name), + }; + + // Three things rule out a reference, and rhai rules out the same three: + // it hands one out for neither a shared cell nor a constant + // (`func/call.rs:1449-1454`), and a function this compiler lowered + // copies its first argument whatever it is handed, exactly as rhai + // copies it before running a script function (`func/call.rs:661`). + let by_reference = place.is_some_and(|value| !is_shared!(value) && !value.is_read_only()) + && program.function(name_index, argc).is_none(); + + // All three want the ordinary shape, with every argument on the stack. + if !by_reference { + // A local's value has not been pushed. A name's already is: it is + // what carried the lookup's position (see [`Receiver::Named`]), and + // it is exactly the value rhai would pass. + if let Site::Slot(index) = at { + let value = scope.get_mut_by_index(index).flatten_clone(); + self.stack.insert(first, value); + } + let value = self.call_stacked(program, name_index, argc, first, pos); + self.stack.truncate(first); + return value; + } + + let value = { + let (entry, rest) = match at { + Site::Slot(index) => (scope.get_mut_by_index(index), first), + // Argument zero is dead weight now that there is an entry to + // reach, and it is the price of having resolved the name where + // its position was. + Site::Name(name) => ( + scope + .get_mut(name) + .ok_or_else(|| malformed(format!("`{name}` stopped being writable")))?, + first + 1, + ), + }; + let mut args: Vec<&mut Dynamic> = core::iter::once(entry) + .chain(self.stack[rest..].iter_mut()) + .collect(); + // The scope a dispatched script function runs in, which is never + // this frame's — see [`Vm::call_stacked`], which has to build one + // for the same reason and cannot borrow this one because the + // receiver is holding it. + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, true, false, &mut args) + .map_err(|err| dispatch_failure(err, pos)) + }; + + self.stack.truncate(first); + value + } + + /// Kept out of the dispatch loop. Inlined, it is enough extra code to + /// change register allocation across every other instruction — measured as + /// a uniform slowdown on benchmarks that call no functions at all. + #[inline(never)] + fn call_compiled( + &mut self, + program: &Program, + name: &str, + params: &[u32], + chunk: crate::grain::bytecode::Chunk, + first: usize, + pos: Position, + ) -> VmResult { + self.engine.track_operation(&mut self.global, pos)?; + + self.global.level += 1; + let result = self.call_compiled_body(program, name, params, chunk, first, pos); + self.global.level -= 1; + result + } + + fn call_compiled_body( + &mut self, + program: &Program, + name: &str, + params: &[u32], + chunk: crate::grain::bytecode::Chunk, + first: usize, + pos: Position, + ) -> VmResult { + if self.global.level > self.engine.max_call_levels() { + return Err(Box::new(EvalAltResult::ErrorStackOverflow(pos))); + } + if params.len() > self.engine.max_variables() { + return Err(Box::new(EvalAltResult::ErrorTooManyVariables(pos))); + } + + // A fresh scope: a function sees its parameters and nothing else. + let mut frame = Scope::new(); + for (param, slot) in params.iter().zip(first..) { + let name = program + .name(*param) + .ok_or_else(|| malformed(format!("no name {param}")))?; + // Taken, not cloned — rhai consumes the caller's argument slots + // (`func/script.rs:75`), and the caller truncates them away after. + let value = self + .stack + .get_mut(slot) + .ok_or_else(|| malformed("call with too few arguments".to_string()))? + .take(); + frame.push_dynamic(name, value); + } + + // A function's parameters are its first locals, sitting at 0 upwards in + // a scope that holds nothing else — so slot 0 is index 0. + let mut reached = chunk.entry() as usize; + let outcome = self.execute(program, &mut frame, chunk, 0, &mut reached); + if outcome.is_err() { + self.fault_pc = Some(reached); + } + + outcome.or_else(|err| match *err { + // A `return` inside the body is the body's value. + EvalAltResult::Return(value, ..) => Ok(value), + // Exits and system errors pass straight through, positioned at the + // call rather than at whatever raised them. + EvalAltResult::Exit(..) => Err(reposition(err, pos)), + _ if err.is_system_exception() => Err(reposition(err, pos)), + // Everything else is attributed to the call. + _ => Err(Box::new(EvalAltResult::ErrorInFunctionCall( + name.to_string(), + self.global.source().unwrap_or("").to_string(), + err, + pos, + ))), + }) + } + + /// Run one frame, cleaning up after it however it leaves. + /// + /// Whatever the frame's loops are holding goes when the frame does — a + /// `return` out of a `for`, or an error escaping one, both skip the + /// `IterNext` that would have dropped the iterator. Doing it here rather + /// than at each exit means there is one place to be right. + fn execute( + &mut self, + program: &Program, + scope: &mut Scope, + chunk: crate::grain::bytecode::Chunk, + base: usize, + reached: &mut usize, + ) -> VmResult { + let iter_base = self.iterators.len(); + let handler_base = self.handlers.len(); + let size_base = self.sizes.len(); + // Each frame's floor is its own. A checkpoint inside a function this + // one calls must not become what this one unwinds to. + let outer_floor = mem::replace(&mut self.unwind_floor, base); + + // The dispatch loop uses `?` throughout, so an error leaves it rather + // than being examined inside it. Catching therefore happens out here: + // the loop stops, a handler this frame armed gets the error, and the + // loop restarts at the catch block. `run_frame` keeps `pc` in a + // register and the fault address arrives through `reached`, which is + // written every instruction anyway, so none of this costs the common + // path anything. + let mut start = chunk.entry() as usize; + let result = loop { + match self.run_frame(program, scope, base, reached, start) { + Ok(value) => break Ok(value), + Err(err) => match self.catch(program, err, handler_base, scope) { + // Metered like a backward jump, and for the same reason: + // a catch block that sits before the throw is a cycle the + // dispatch loop never sees as one, because control got + // there through the error path rather than through a jump. + Ok(resume) => { + self.engine + .track_operation(&mut self.global, program.position(resume))?; + start = resume; + } + Err(err) => break Err(err), + }, + } + }; + + self.iterators.truncate(iter_base); + self.handlers.truncate(handler_base); + self.sizes.truncate(size_base); + + if result.is_err() { + self.unwind_after_error(scope); + } + self.unwind_floor = outer_floor; + result + } + + /// Rhai's `Engine::make_type_mismatch_err` (`api/formatting.rs:246`). + /// + /// The asymmetry is rhai's and is easy to get wrong in either direction: + /// the *expected* type goes through the engine's registered names and the + /// *actual* one does not. So `if 0..1 {}` reports + /// `core::ops::range::Range` rather than the `range` the same engine + /// would print anywhere else. Mapping both — which reads like the obvious + /// thing — makes every one of these differ from the walker. + fn mismatch(&self, actual: &str, pos: Position) -> Box { + Box::new(EvalAltResult::ErrorMismatchDataType( + self.engine + .map_type_name(core::any::type_name::()) + .into(), + actual.into(), + pos, + )) + } + + /// What reading a key a map does not have produces. + /// + /// Unit, unless the host asked for the strict reading — which is a whole + /// engine option (`fail_on_invalid_map_property`) rather than anything the + /// script says, so it has to be consulted rather than assumed. + fn absent_key(&self, key: &str, pos: Position) -> VmResult { + if self.engine.fail_on_invalid_map_property() { + Err(Box::new(EvalAltResult::ErrorPropertyNotFound( + key.to_string(), + pos, + ))) + } else { + Ok(Dynamic::UNIT) + } + } + + /// Fold the element on top of the stack into the literal's running total, + /// and refuse it if that puts the literal over a configured limit. + /// + /// Reproduces `eval/expr.rs:318-329` for an array and `:349-359` for a map. + /// The two differ in one place — an array element adds one to the array + /// count, a map entry adds one to the map count — and in nothing else, so + /// they share this. + /// + /// Worth being exact about what is *not* counted: a map's total starts at + /// zero and only the entries with computed values are added to it, because + /// rhai's loop runs over those alone and the constant ones are already + /// sitting in the template. A literal that is entirely constant never + /// reaches here at all — the optimizer folded it long before. + /// + /// Out of line: it is a handful of instructions in the common case and a + /// call to rhai in the rare one, and the dispatch loop is measurably + /// sensitive to what shares its registers. + #[inline(never)] + fn check_size( + &mut self, + index: u16, + map: bool, + pos: Position, + ) -> Result<(), Box> { + if index == 0 { + self.sizes.push((0, 0, 0)); + } + + // Rhai skips the whole measurement when no limit could reject it, and + // measuring is a walk of the value — so this is the difference between + // free and proportional to what the literal holds. + if self.engine.max_string_size() == 0 + && self.engine.max_array_size() == 0 + && self.engine.max_map_size() == 0 + { + return Ok(()); + } + + let value = self + .stack + .last() + .ok_or_else(|| malformed("size check with no element".to_string()))?; + let delta = calc_data_sizes(value, true); + + let total = self + .sizes + .last_mut() + .ok_or_else(|| malformed("size check outside a literal".to_string()))?; + *total = ( + total.0 + delta.0 + usize::from(!map), + total.1 + delta.1 + usize::from(map), + total.2 + delta.2, + ); + + self.engine + .throw_on_size(*total) + .map_err(|err| positioned(err, pos)) + } + + /// Drop what an escaping error skipped the unwind for. + /// + /// An error leaves a block by jumping over the [`Op::UnwindTo`] that would + /// have dropped what it declared, so those locals are still in the scope. + /// Rhai rewinds a block whether it is left normally or by a throw, and + /// rewinds nothing at a chunk's top level — which is what the floor is: the + /// last top-level statement boundary. Anything above it belongs to a block + /// that did not get to finish. + /// + /// Guarded rather than unconditional because [`Op::Return`] has already + /// unwound to `base`, which is below the floor. + /// + /// Out of line for the reason [`Vm::catch`] is: it sits on the error edge + /// of the frame, where nothing is hot and everything competes with the + /// dispatch loop for the same registers. + #[inline(never)] + fn unwind_after_error(&self, scope: &mut Scope) { + if scope.len() > self.unwind_floor { + scope.rewind(self.unwind_floor); + } + } + + /// Hand an error to the innermost handler this frame armed, if any. + /// + /// `Ok` is the address the catch block starts at. `Err` means nothing here + /// wanted it and it should keep going up. + /// + /// Kept out of line for the reason [`Vm::call_compiled`] is: it sits on + /// the dispatch loop's error edge, and letting it inline there costs every + /// instruction that never fails. + #[inline(never)] + fn catch( + &mut self, + program: &Program, + err: Box, + handler_base: usize, + scope: &mut Scope, + ) -> Result> { + // Only handlers this frame armed. A callee must never resume into its + // caller's catch block — that is a jump into another chunk, which the + // verifier forbids and nothing would catch at run time. The callee's + // error propagates normally instead, and `ErrorInFunctionCall` is + // catchable, so the caller's own frame still sees it. + // Walking outwards, because leaving one region can land the error in + // the next: `try { try { throw 1 } catch { throw; } } catch (e) { .. }` + // re-raises from the inner catch and the outer `try` still has to see + // it. + let mut err = err; + let handler = loop { + if self.handlers.len() <= handler_base { + return Err(err); + } + let handler = self.handlers.last_mut().expect("checked"); + + // Leaving a catch block rather than entering one. A bare `throw;` + // there — an `ErrorRuntime` carrying unit — means "re-raise what + // was caught, from here" (`eval/stmt.rs:866`). + let Some(original) = handler.caught.take() else { + break handler; + }; + self.handlers.pop(); + let rethrown = + matches!(&*err, EvalAltResult::ErrorRuntime(value, ..) if value.is_unit()); + if rethrown { + let pos = err.position(); + err = original; + err.set_position(pos); + } + }; + + // `return`, `break`, `continue`, `exit` and the system exceptions + // unwind as errors and are not exceptions a script may catch. + if !err.is_catchable() { + return Err(err); + } + + let (target, catch_var) = (handler.target, handler.catch_var); + let (operands, scope_len, iters) = (handler.operands, handler.scope_len, handler.iters); + + let mut err = err; + let value = self.catch_value(&mut err, catch_var.is_some()); + + // Back to where the `try` began, at all three depths. + self.stack.truncate(operands); + self.iterators.truncate(iters); + scope.rewind(scope_len); + + if let Some(index) = catch_var { + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + if scope.len() >= self.engine.max_variables() { + return Err(Box::new(EvalAltResult::ErrorTooManyVariables( + program.position(target), + ))); + } + scope.push_dynamic(name, value); + } + + self.handlers.last_mut().expect("checked").caught = Some(err); + Ok(target) + } + + /// What the catch variable is bound to (`eval/stmt.rs:809-845`). + /// + /// Three shapes: nothing at all without a variable, the raw thrown value + /// for a `throw`, and a map of the error's parts for anything else. The + /// unwrapping matters — a `throw` inside a called function arrives wrapped + /// in `ErrorInFunctionCall`, and rhai still binds the bare value. + fn catch_value(&self, err: &mut Box, wanted: bool) -> Dynamic { + if !wanted { + return Dynamic::UNIT; + } + if let EvalAltResult::ErrorRuntime(value, ..) = err.unwrap_inner() { + return value.clone(); + } + + let mut map = Map::new(); + // Read *and cleared*, as rhai does, so the message below carries no + // trailing position and a re-raise starts from the catch site. + let pos = err.take_position(); + + map.insert("message".into(), err.to_string().into()); + if let Some(source) = &self.global.source { + map.insert("source".into(), source.into()); + } + if !pos.is_none() { + let line = pos.line().unwrap_or(0) as INT; + map.insert("line".into(), line.into()); + let column = pos.position().unwrap_or(0) as INT; + map.insert("position".into(), column.into()); + } + err.dump_fields(&mut map); + map.into() + } + + /// The dispatch loop. `start` is the chunk's entry, or a catch block's + /// address when [`Vm::execute`] resumes one after an error. + /// + /// Inlined into its one caller: splitting the loop out so errors could be + /// caught outside it cost 1.55x to 1.40x on the tight-loop benchmark until + /// this was here. + #[inline(always)] + fn run_frame( + &mut self, + program: &Program, + scope: &mut Scope, + base: usize, + reached: &mut usize, + start: usize, + ) -> VmResult { + // A called function pushes its operands above the caller's rather than + // starting a stack of its own, so this records where its own begin. + let stack_base = self.stack.len(); + self.stack.reserve(program.max_stack() as usize); + + // A residual's `Expr::Variable` nodes carry offsets rhai's parser + // computed against its own scope discipline, not against ours. Forcing + // name lookup inside them costs a reverse scan but cannot be wrong. + // Only programs that still have residuals pay it, which is the point of + // driving the count to zero. + if program.residual_count() > 0 { + self.global.always_search_scope = true; + } + + let code = program.code(); + // The chunk's entry the first time round, a catch block's address when + // resumed after one. + let mut pc = start; + + loop { + // Nothing inside an iteration moves `pc` except a jump, and a jump + // only happens after the instruction succeeded — so recording it + // here names whichever instruction fails. + *reached = pc; + + // No check against the chunk's end. Verification proves execution + // cannot leave it — every path reaches a `Return`, no jump goes + // outside, nothing falls off — so a comparison here would cost + // every instruction to restate something already established. + let tag = *code.get(pc).ok_or_else(|| { + Box::new(EvalAltResult::ErrorRuntime( + format!("ran off the end of a chunk at {pc}").into(), + Position::NONE, + )) + })?; + + // Every instruction's operands sit at a fixed offset from its tag, + // so dispatch is a match and a couple of loads with nothing decoded + // and nothing allocated. The bounds checks are what let this run + // straight off an artifact without trusting it; the verifier has + // already made them unreachable for anything that loaded. + let width = code::width(code, pc) + .ok_or_else(|| malformed(format!("undecodable instruction at {pc}")))?; + let small = |offset: usize| { + code::u16_at(code, pc + offset) + .ok_or_else(|| malformed(format!("truncated operand at {pc}"))) + }; + let wide = |offset: usize| { + code::u32_at(code, pc + offset) + .ok_or_else(|| malformed(format!("truncated operand at {pc}"))) + }; + + // Instructions carry no position; the table does, keyed on the + // address. A stripped program answers `NONE` for every one of + // these, which is what a device runs — the address travels back + // with the error instead, and the host resolves it. + // + // A closure rather than a value: most instructions never ask, and + // the ones that do mostly ask only on the way to an error. + let pos = || program.position(pc); + + // Every transfer of control goes through this, and a backward one + // is charged an operation. + // + // A cycle in a chunk always contains a backward edge, so this is + // what makes `max_operations` and the `on_progress` interrupt cover + // a chunk *this compiler did not write*. `Op::Tick` covers the + // loops it does write, positioned where rhai would report them; a + // corrupt artifact has no ticks at all and would otherwise spin + // forever inside a loader that had already accepted it. Found by + // `mutated_artifacts_load_or_fail_but_never_misbehave`, whose whole + // claim is that this cannot happen. + // + // A macro rather than four open-coded checks because the failure + // mode of missing one is silent, and because it costs nothing on + // the straight-line path: only a jump pays the comparison. + macro_rules! transfer { + ($target:expr) => {{ + let target: usize = $target; + if target <= pc { + self.engine.track_operation(&mut self.global, pos())?; + } + pc = target; + }}; + } + + match tag { + code::tag::CONST => { + let index = u32::from(small(1)?); + let value = program + .constant(index) + .ok_or_else(|| malformed(format!("no constant {index}")))?; + self.stack.push(value.clone()); + } + + code::tag::UNIT => self.stack.push(Dynamic::UNIT), + code::tag::FALSE => self.stack.push(Dynamic::from(false)), + code::tag::TRUE => self.stack.push(Dynamic::from(true)), + + code::tag::LOAD_LOCAL => { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + // Reads clone out, matching how rhai's own variable reads + // leave the scope entry alone (`eval/expr.rs:276-278`), and + // flattening any shared cell the way a read should. + self.stack + .push(scope.get_mut_by_index(index).flatten_clone()); + } + + code::tag::STORE_LOCAL => { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + let value = self.pop()?; + // Through the cell, not over it — see `place`. + *place(scope.get_mut_by_index(index), "", pos())? = value; + } + + code::tag::LOAD_NAMED | code::tag::LOAD_SHARED_NAMED => { + let index = u32::from(small(1)?); + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + let flatten = tag == code::tag::LOAD_NAMED; + let value = self.load_named(name, scope, flatten, pos())?; + self.stack.push(value); + } + + code::tag::ASSIGN_NAMED | code::tag::ASSIGN_NAMED_OP => { + let index = u32::from(small(1)?); + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + let op = if tag == code::tag::ASSIGN_NAMED_OP { + let index = u32::from(small(3)?); + Some( + program + .assign_op(index) + .ok_or_else(|| malformed(format!("no op-assignment {index}")))?, + ) + } else { + None + }; + + // Flattened before assigning, as rhai does, so a shared + // cell is copied out rather than aliased into the target. + let rhs = self.pop()?.flatten(); + self.assign_named(program, op, name, rhs, scope, pos())?; + } + + code::tag::DECLARE_LOCAL | code::tag::DECLARE_CONST => { + let index = u32::from(small(1)?); + // A `Scope` entry name is an `Identifier`, which is a + // `SmartString` — short names live inline, so handing it a + // borrowed `&str` costs a copy rather than an allocation. + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + let value = self.pop()?; + if tag == code::tag::DECLARE_CONST { + scope.push_constant_dynamic(name, value); + } else { + scope.push_dynamic(name, value); + } + } + + code::tag::ASSIGN_LOCAL | code::tag::ASSIGN_LOCAL_OP => { + let slot = small(1)?; + let var_name = u32::from(small(3)?); + let op = if tag == code::tag::ASSIGN_LOCAL_OP { + let index = u32::from(small(5)?); + Some( + program + .assign_op(index) + .ok_or_else(|| malformed(format!("no op-assignment {index}")))?, + ) + } else { + None + }; + + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + + // Rhai flattens the right-hand side before assigning + // (`eval/stmt.rs:324`), so a shared cell is copied out + // rather than aliased into the target. + let rhs = self.pop()?.flatten(); + + if scope.get_mut_by_index(index).is_read_only() { + let name = program + .name(var_name) + .ok_or_else(|| malformed(format!("no name {var_name}")))?; + return Err(Box::new(EvalAltResult::ErrorAssignmentToConstant( + name.to_string(), + pos(), + ))); + } + + // Written through rather than over: a slot a closure + // captured is a shared cell, and replacing it would sever + // every holder. `store` is the same path a chain's tail + // and a named assignment take, so `x op= y` resolves + // identically wherever the target lives. + let name = program + .name(var_name) + .ok_or_else(|| malformed(format!("no name {var_name}")))?; + // The guard is only needed for a cell a closure captured, + // and `x += 1` in a loop is the hot path — so the check + // for one is a discriminant test rather than the downcast + // chain `write_lock` walks, and the built-in operator is + // reached without leaving the dispatch loop or resolving + // the position. + // + // It is not free even so: the tight-loop benchmark went + // 1.63x to 1.55x when locals stopped being written over + // and started being written through. That is the price of + // a shared cell surviving an assignment, and of a chain + // over one not taking the host down. + let entry = scope.get_mut_by_index(index); + if !is_shared!(entry) { + let mut rhs = rhs; + if let Some(done) = + op.and_then(|op| self.store_builtin(op, entry, &mut rhs, pos)) + { + done?; + pc += width; + continue; + } + self.store(program, op, entry, rhs, pos())?; + pc += width; + continue; + } + + let mut target = place(entry, name, pos())?; + self.store(program, op, &mut target, rhs, pos())?; + } + + code::tag::POP => { + let _ = self.pop()?; + } + + code::tag::EVAL_AST | code::tag::EVAL_AST_KEEP => { + let index = u32::from(small(1)?); + let expr = program + .residual(index) + .ok_or_else(|| malformed(format!("no residual {index}")))?; + let rewind_scope = tag == code::tag::EVAL_AST; + + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + scope, + None, + ); + + // The deprecation marker on this method means "volatile, + // may change", not "going away" — it is the only public + // route from outside the crate to rhai's own walker, and + // total language coverage rests on it. + #[allow(deprecated)] + let value = + context.eval_expression_tree_raw(&Expression::from(expr), rewind_scope)?; + + self.stack.push(value); + } + + code::tag::JUMP => { + transfer!(wide(1)? as usize); + continue; + } + + code::tag::JUMP_IF_FALSE | code::tag::JUMP_IF_TRUE => { + let target = wide(1)? as usize; + let condition = self.pop()?; + // Rhai requires a boolean guard and reports the mismatch at + // the guard's own position (`eval/stmt.rs:487-490`). + let holds = condition + .as_bool() + .map_err(|actual| self.mismatch::(actual, pos()))?; + if holds == (tag == code::tag::JUMP_IF_TRUE) { + transfer!(target); + continue; + } + } + + code::tag::CALL | code::tag::CALL_OP => { + let name_index = u32::from(small(1)?); + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + let argc = code[pc + 3] as usize; + let op = if tag == code::tag::CALL_OP { + let index = u32::from(small(4)?); + Some( + program + .token(index) + .ok_or_else(|| malformed(format!("no operator {index}")))?, + ) + } else { + None + }; + + let first = self + .stack + .len() + .checked_sub(argc) + .ok_or_else(|| malformed("call with too few arguments".to_string()))?; + + // Reach the same built-in the walker reaches. Gated on + // rhai's own `fast_operators()` rather than a guard of our + // own, so an engine that turns it off gets the dispatch + // path on both sides, and one that leaves it on gets the + // same answer — including for a user-registered operator + // on a primitive, which rhai's fast path also bypasses + // (`func/call.rs:1775-1799`). + if let (Some(token), 2, true) = (op, argc, self.engine.fast_operators()) { + let (lhs, rhs) = self.stack.split_at_mut(first + 1); + let lhs = &mut lhs[first]; + let rhs = &mut rhs[0]; + + // Custom types go to dispatch first, so a registered + // function still wins for them. + let builtin = (!lhs.is_variant() && !rhs.is_variant()) + .then(|| get_builtin_binary_op_fn(token, lhs, rhs)) + .flatten(); + if let Some((func, need_context)) = builtin { + let context = need_context.then(|| { + native_context(self.engine, name, None, &self.global, pos()) + }); + let value = func(context, &mut [lhs, rhs])?; + self.stack.truncate(first); + self.stack.push(value); + pc += width; + continue; + } + } + + let value = self.call_stacked(program, name_index, argc, first, pos())?; + self.stack.truncate(first); + self.stack.push(value); + } + + code::tag::CALL_LOCAL_REF | code::tag::CALL_NAMED_REF => { + let name_index = u32::from(small(1)?); + let argc = code[pc + 3] as usize; + let receiver = if tag == code::tag::CALL_LOCAL_REF { + Receiver::Local(small(4)?) + } else { + Receiver::Named(u32::from(small(4)?)) + }; + + let value = self.call_by_reference( + program, + name_index, + argc, + receiver, + scope, + base, + pos(), + )?; + self.stack.push(value); + } + + code::tag::ROTATE => { + let under = code[pc + 1] as usize; + let top = self + .stack + .len() + .checked_sub(1) + .ok_or_else(|| malformed("rotate on an empty stack".to_string()))?; + let to = top + .checked_sub(under) + .ok_or_else(|| malformed("rotate past the bottom".to_string()))?; + self.stack[to..].rotate_right(1); + } + + code::tag::MAKE_ARRAY => { + let len = small(1)? as usize; + let first = self + .stack + .len() + .checked_sub(len) + .ok_or_else(|| malformed("array with too few elements".to_string()))?; + + // The running total belongs to this literal and goes with + // it. `Op::CheckSize` is what filled it in, one element at + // a time, and what raised `ErrorDataTooLarge` against the + // element that tipped it over (`eval/expr.rs:307-330`). + // + // Only if there was one: an empty literal emits no + // `CheckSize` and pushed nothing, so popping here would + // take the *enclosing* literal's total — `[a, [], b]`. + if len > 0 { + self.sizes.pop(); + } + + // Flattened, as rhai does, so a shared cell is copied in + // rather than aliased. + let array: Array = self.stack.drain(first..).map(Dynamic::flatten).collect(); + self.stack.push(Dynamic::from_array(array)); + } + + code::tag::MAKE_MAP => { + let len = small(1)? as usize; + let first = self + .stack + .len() + .checked_sub(2 * len + 1) + .ok_or_else(|| malformed("map with too few operands".to_string()))?; + // As for `MakeArray`: nothing was pushed for a literal + // with no computed entries, so nothing may be popped. + if len > 0 { + self.sizes.pop(); + } + + let mut parts = self.stack.drain(first..); + let template = parts.next().expect("checked above"); + let mut map = template + .try_cast::() + .ok_or_else(|| malformed("map literal without a template".to_string()))?; + while let Some(key) = parts.next() { + let value = parts.next().expect("pairs, checked above"); + let key = key.into_immutable_string().map_err(|actual| { + malformed(format!("map key is a {actual}, not a string")) + })?; + // Flattened as rhai does, so a shared cell is copied + // in rather than aliased. + map.insert(key.as_str().into(), value.flatten()); + } + drop(parts); + self.stack.push(Dynamic::from_map(map)); + } + + code::tag::CHECK_ARRAY_SIZE | code::tag::CHECK_MAP_SIZE => { + let index = small(1)?; + let map = tag == code::tag::CHECK_MAP_SIZE; + self.check_size(index, map, pos())?; + } + + code::tag::SWITCH => { + let index = u32::from(small(1)?); + let table = program + .switch(index) + .ok_or_else(|| malformed(format!("no switch {index}")))?; + let subject = self.pop()?; + // Always a jump: an arm that matched nothing still has the + // default to go to. + transfer!(table.dispatch(&subject) as usize); + continue; + } + + code::tag::LOAD_SHARED => { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + // Cloned, not flattened: cloning a shared `Dynamic` clones + // the `Rc`, which is the capture. + self.stack.push(scope.get_mut_by_index(index).clone()); + } + + // Emitted only for a closure capture, which cannot be parsed + // under `no_closure`. + #[cfg(not(feature = "no_closure"))] + code::tag::SHARE | code::tag::SHARE_NAMED => { + let entry = if tag == code::tag::SHARE { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + Some(index) + } else { + let name_index = u32::from(small(1)?); + let name = program + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; + // The resolver gets first refusal, and a name it + // answers is not shared at all (`eval/stmt.rs:998`). + if self.resolve_var(name, scope, pos())?.is_some() { + pc += width; + continue; + } + // `iter_raw` walks the scope from the top down, which is + // the order shadowing wants — the first match is the + // live one — but it counts from the other end than + // `get_mut_by_index` does, so the position has to be + // turned back round. Rhai reaches the same entry + // through `Scope::search`, which is not public + // (`eval/stmt.rs:1009`). + let depth = scope.len(); + let found = scope + .iter_raw() + .position(|(entry, ..)| entry == name) + .map(|from_top| depth - 1 - from_top); + Some(found.ok_or_else(|| missing(name, pos()))?) + }; + + if let Some(index) = entry { + let value = scope.get_mut_by_index(index); + if !value.is_shared() { + *value = value.take().into_shared(); + } + } + } + + code::tag::MAKE_CLOSURE => { + let index = u32::from(small(1)?); + let name = program + .name(index) + .ok_or_else(|| malformed(format!("no name {index}")))?; + // Unvalidated, because `anon$…` is not a name a script could + // have written and the validating constructors refuse it. + // Nothing unsound rides on that check — a name that will + // not resolve simply fails when the pointer is called. + self.stack.push( + FnPtr { + name: name.into(), + curry: ThinVec::new(), + #[cfg(not(feature = "no_function"))] + env: None, + typ: FnPtrType::Normal, + } + .into(), + ); + } + + #[cfg(not(feature = "no_closure"))] + code::tag::IS_SHARED => { + let value = self.pop()?; + self.stack.push(value.is_shared().into()); + } + + code::tag::MAKE_FN_PTR => { + let name = self.pop()?; + let name = name + .into_immutable_string() + .map_err(|actual| self.mismatch::(actual, pos()))?; + // Validates that the name is an identifier, as rhai's own + // `Fn(..)` does (`func/call.rs:1215`). + let pointer = FnPtr::new(name).map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos()); + } + err + })?; + self.stack.push(pointer.into()); + } + + code::tag::CURRY => { + let argc = code[pc + 1] as usize; + let at = self + .stack + .len() + .checked_sub(argc + 1) + .ok_or_else(|| malformed("curry is missing its target".into()))?; + let mut pointer = self.stack[at] + .clone() + .try_cast::() + .ok_or_else(|| self.mismatch::(self.stack[at].type_name(), pos()))?; + for value in self.stack.drain(at + 1..) { + pointer.add_curry(value); + } + self.stack.truncate(at); + self.stack.push(pointer.into()); + } + + code::tag::CALL_FN_PTR | code::tag::CALL_FN_PTR_METHOD => { + let argc = code[pc + 1] as usize; + let method = tag == code::tag::CALL_FN_PTR_METHOD; + let value = self.call_fn_ptr(program, argc, method, pos())?; + self.stack.push(value); + } + + code::tag::INTERPOLATE_START => { + self.stack.push(self.engine.const_empty_string().into()); + } + + code::tag::INTERPOLATE_APPEND => { + let segment = self.pop()?; + self.append_segment(segment, pos())?; + } + + code::tag::INTERPOLATE_END => { + let buffer = self.pop()?; + let text = buffer + .into_immutable_string() + .map_err(|_| malformed("interpolation lost its buffer".into()))?; + // Interned, as rhai does: the same rendered string in ten + // places is one allocation, which is the whole reason the + // engine keeps an interner. + let value = self.engine.get_interned_string(text.as_str()); + self.stack.push(value.into()); + } + + code::tag::CHAIN => { + let index = u32::from(small(1)?); + let chain = program + .chain(index) + .ok_or_else(|| malformed(format!("no chain {index}")))?; + let value = self.run_chain(program, chain, scope, base, pos())?; + self.stack.push(value); + } + + code::tag::UNWIND_TO => { + let depth = small(1)?; + let target = base + depth as usize; + if target > scope.len() { + return Err(malformed(format!( + "unwind to {target} past a scope of {}", + scope.len() + ))); + } + scope.rewind(target); + } + + code::tag::TICK => self.engine.track_operation(&mut self.global, pos())?, + + code::tag::CHECKPOINT => self.unwind_floor = scope.len(), + + code::tag::PUSH_HANDLER | code::tag::PUSH_HANDLER_VAR => { + let target = wide(1)? as usize; + let catch_var = if tag == code::tag::PUSH_HANDLER_VAR { + Some(u32::from(small(5)?)) + } else { + None + }; + self.handlers.push(Handler { + target, + catch_var, + operands: self.stack.len(), + scope_len: scope.len(), + iters: self.iterators.len(), + caught: None, + }); + } + + code::tag::POP_HANDLER => { + self.handlers.pop(); + } + + code::tag::ITER_INIT => { + let iterable = self.pop()?; + self.iter_init(iterable, pos())?; + } + + code::tag::ITER_DROP => { + self.iterators.pop(); + } + + code::tag::ITER_NEXT | code::tag::ITER_NEXT_INDEXED => { + let exit = wide(1)? as usize; + let iteration = self + .iterators + .last_mut() + .ok_or_else(|| malformed("no iterator to advance".to_string()))?; + + let Some(item) = iteration.items.next() else { + self.iterators.pop(); + transfer!(exit); + continue; + }; + + // Counted before the item is unwrapped, as rhai does, so a + // loop long enough to wrap the counter is an error rather + // than a wrap. + iteration.count = iteration.count.checked_add(1).ok_or_else(|| { + Box::new(EvalAltResult::ErrorArithmetic( + format!("for-loop counter overflow: {}", iteration.count), + pos(), + )) + })?; + let count = iteration.count; + + // A fallible iterator's error is positioned at the + // iterable, and only if it brought none of its own + // (`eval/stmt.rs:749`). + let value = item.map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos()); + } + err + })?; + + if tag == code::tag::ITER_NEXT_INDEXED { + self.stack.push(Dynamic::from(count)); + } + self.stack.push(value.flatten()); + } + + code::tag::STORE_SHARED => { + let slot = small(1)?; + let index = base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + let value = self.pop()?; + // Through the cell: a closure made in an earlier iteration + // shares this slot, and rhai writes into it rather than + // replacing it (`eval/stmt.rs:752`). + *place(scope.get_mut_by_index(index), "", pos())? = value; + } + + code::tag::THROW => { + // Flattened, as rhai does, so a shared cell is thrown as + // its value rather than as the cell. + let value = self.pop()?.flatten(); + return Err(Box::new(EvalAltResult::ErrorRuntime(value, pos()))); + } + + code::tag::RETURN => { + let value = self.stack.pop().unwrap_or(Dynamic::UNIT); + // Whatever else this frame left behind goes with it, so a + // caller's stack is exactly as it was. + self.stack.truncate(stack_base); + return Ok(value); + } + + // `code::width` already refused anything it does not know, so + // this is unreachable — but a wildcard is what stops a new tag + // from silently falling through to the next instruction. + _ => return Err(malformed(format!("unknown instruction {tag:#04x} at {pc}"))), + } + + pc += width; + } + } +} diff --git a/src/lib.rs b/src/lib.rs index e40ea3748..0e611246b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,6 +121,8 @@ pub mod config; mod engine; mod eval; mod func; +#[cfg(feature = "grain")] +pub mod grain; mod module; mod optimizer; pub mod packages; diff --git a/src/module/mod.rs b/src/module/mod.rs index a216d4c08..cde9a3024 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -373,10 +373,10 @@ impl FuncRegistration { /// /// # Parameter Examples /// - /// `"foo: &str"` <- parameter name = `foo`, type = `&str` - /// `"bar"` <- parameter name = `bar`, type unknown - /// `"_: i64"` <- parameter name unknown, type = `i64` - /// `"MyType"` <- parameter name unknown, type = `MyType` + /// `"foo: &str"` <- parameter name = `foo`, type = `&str` + /// `"bar"` <- parameter name = `bar`, type unknown + /// `"_: i64"` <- parameter name unknown, type = `i64` + /// `"MyType"` <- parameter name unknown, type = `MyType` #[cfg(feature = "metadata")] #[must_use] pub fn with_params_info>(mut self, params: impl IntoIterator) -> Self { diff --git a/tests/grain/allocation.rs b/tests/grain/allocation.rs new file mode 100644 index 000000000..fb87acce2 --- /dev/null +++ b/tests/grain/allocation.rs @@ -0,0 +1,385 @@ +//! What the AST costs, and what the VM costs instead. +//! +//! This is the instrument the project's premise rests on. Rhai's tree costs +//! ~24 bytes of device heap per minified source byte, and traffic-light's +//! `firmware/src/script.rs:88-92` records something the retained figure misses: +//! "rhai's parser allocates well past the size of the tree it finally keeps, so +//! the peak during `compile` is what has to fit, not the result." So this +//! tracks peak as well as retained — traffic-light's own harness tracks only +//! retained, and peak is the number that actually bounds script size. +//! +//! # Why this is one test in its own file +//! +//! The counters are process-global, and cargo runs tests on parallel threads. +//! Any concurrent allocation lands in the same counters and corrupts every +//! reading. One `#[test]`, in its own binary, is what keeps the numbers real. +//! traffic-light learned the same thing (commit bee5fb5, "Fold the allocation +//! measurement into the one test that owns the counters"). +//! +//! # What these numbers are not +//! +//! Host figures. They do not transfer to a device by halving: traffic-light +//! measured the engine ratio at 0.71 rather than 0.5, and found the AST does +//! not shrink on 32-bit at all, because `Dynamic`, `i64` and `f32` fields are +//! the same width either way. Device numbers come from the device, in M6. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicIsize, Ordering}; + +use rhai::Engine; +use rhaigrain::Compiler; + +static LIVE: AtomicIsize = AtomicIsize::new(0); +static COUNT: AtomicIsize = AtomicIsize::new(0); +static PEAK: AtomicIsize = AtomicIsize::new(0); + +/// Live allocations by size class, `SIZE_CLASSES[i-1] < size <= SIZE_CLASSES[i]`. +/// The small classes are what a per-allocation header punishes, and an AST is +/// mostly small classes. +static BUCKETS: [AtomicIsize; 6] = [ + AtomicIsize::new(0), + AtomicIsize::new(0), + AtomicIsize::new(0), + AtomicIsize::new(0), + AtomicIsize::new(0), + AtomicIsize::new(0), +]; +const SIZE_CLASSES: [usize; 6] = [8, 16, 32, 64, 256, usize::MAX]; + +fn bucket_of(size: usize) -> usize { + SIZE_CLASSES.iter().position(|&c| size <= c).unwrap_or(5) +} + +struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + let now = LIVE.fetch_add(l.size() as isize, Ordering::Relaxed) + l.size() as isize; + PEAK.fetch_max(now, Ordering::Relaxed); + COUNT.fetch_add(1, Ordering::Relaxed); + BUCKETS[bucket_of(l.size())].fetch_add(1, Ordering::Relaxed); + unsafe { System.alloc(l) } + } + + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + LIVE.fetch_sub(l.size() as isize, Ordering::Relaxed); + COUNT.fetch_sub(1, Ordering::Relaxed); + BUCKETS[bucket_of(l.size())].fetch_sub(1, Ordering::Relaxed); + unsafe { System.dealloc(p, l) } + } + + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + let now = LIVE.fetch_add(new as isize - l.size() as isize, Ordering::Relaxed) + new as isize + - l.size() as isize; + PEAK.fetch_max(now, Ordering::Relaxed); + BUCKETS[bucket_of(l.size())].fetch_sub(1, Ordering::Relaxed); + BUCKETS[bucket_of(new)].fetch_add(1, Ordering::Relaxed); + unsafe { System.realloc(p, l, new) } + } +} + +#[global_allocator] +static ALLOC: Counting = Counting; + +fn live() -> isize { + LIVE.load(Ordering::Relaxed) +} + +fn count() -> isize { + COUNT.load(Ordering::Relaxed) +} + +fn buckets() -> [isize; 6] { + std::array::from_fn(|i| BUCKETS[i].load(Ordering::Relaxed)) +} + +/// What producing `T` cost, and what it goes on costing while held. +struct Measured { + value: T, + /// Still-live bytes attributable to the returned value. + bytes: isize, + /// Still-live allocations attributable to the returned value. + count: isize, + /// High-water mark during the call. Exceeds `bytes` by whatever the + /// producer allocated and freed along the way — for rhai's parser, a lot. + peak: isize, + /// Still-live allocations by size class. + buckets: [isize; 6], +} + +fn measure(f: impl FnOnce() -> T) -> Measured { + let base_bytes = live(); + let base_count = count(); + let base_buckets = buckets(); + PEAK.store(base_bytes, Ordering::Relaxed); + + let value = f(); + + let peak = PEAK.load(Ordering::Relaxed) - base_bytes; + let after = buckets(); + + Measured { + bytes: live() - base_bytes, + count: count() - base_count, + peak, + buckets: std::array::from_fn(|i| after[i] - base_buckets[i]), + value, + } +} + +/// Stands in for a real control script until M1 wires in traffic-light's +/// `scripts/follow.rhai`, which is what the 24 bytes/source-byte figure was +/// measured against. Shaped like one: helper functions, integer and float +/// arithmetic, arrays, and a loop driving them. +const SCRIPT: &str = r#" +fn clamp(v, lo, hi) { + if v < lo { lo } else if v > hi { hi } else { v } +} + +fn ease(t) { + let x = clamp(t, 0.0, 1.0); + x * x * (3.0 - 2.0 * x) +} + +fn blend(a, b, t) { + let k = ease(t); + (a * (1.0 - k) + b * k) +} + +let channels = [0, 0, 0, 0, 0, 0, 0, 0]; +let phase = 0.0; +let step = 0.125; + +for frame in 0..64 { + phase += step; + if phase > 1.0 { phase -= 1.0; } + + for i in 0..channels.len { + let target = if i % 2 == 0 { 255.0 } else { 64.0 }; + let level = blend(0.0, target, ease(phase)); + channels[i] = clamp(level.to_int(), 0, 255); + } +} + +channels +"#; + +/// The script the premise is actually about — byte-identical to +/// traffic-light's `scripts/follow.rhai`, the one its 24 bytes/source-byte +/// figure was measured against. +const FOLLOW: &str = include_str!("fixtures/follow.rhai"); + +/// Something that lowers all the way, so it can be written and read back. +/// +/// `follow.rhai` cannot yet: it still fragments on property reads and index +/// assignment, and a program holding a fragment refuses to serialize. Until +/// chained lvalues land, the load figures come from a script shaped like the +/// parts of it that do lower — locals, arithmetic, branches and a loop. +const LOADABLE: &str = r#" +let total = 0; +let phase = 0; +let step = 3; +let i = 0; +while i < 64 { + phase += step; + if phase > 100 { phase -= 100; } + if i % 2 == 0 { total += phase * 2; } else { total -= phase; } + i += 1; +} +total; +"#; + +#[test] +fn allocation_footprint() { + let engine = Engine::new(); + + let engine_cost = measure(Engine::new); + let ast = measure(|| engine.compile(SCRIPT).expect("script must compile")); + let program = measure(|| Compiler::new().compile(&ast.value)); + + let follow_ast = measure(|| engine.compile(FOLLOW).expect("follow.rhai must compile")); + + let source_bytes = SCRIPT.len() as isize; + let per_source_byte = ast.bytes as f64 / source_bytes as f64; + + println!("\nsource {source_bytes} bytes"); + println!( + "\n{:<24} {:>10} {:>10} {:>10}", + "", "bytes", "allocs", "peak" + ); + println!( + "{:<24} {:>10} {:>10} {:>10}", + "Engine::new", engine_cost.bytes, engine_cost.count, engine_cost.peak + ); + println!( + "{:<24} {:>10} {:>10} {:>10}", + "engine.compile (AST)", ast.bytes, ast.count, ast.peak + ); + println!( + "{:<24} {:>10} {:>10} {:>10}", + "Compiler::compile", program.bytes, program.count, program.peak + ); + + // Not yet comparable to traffic-light's 24.0: that is a 32-bit device + // figure measured against minified source, this is a host figure against + // unminified source. M1 makes them comparable by measuring follow.rhai, + // minified, the way the server actually ships it. + println!("\nAST bytes per source byte {per_source_byte:.1} (host, unminified)"); + println!( + "AST parser peak / retained {:.2}x (the peak is what has to fit)", + ast.peak as f64 / ast.bytes as f64 + ); + + println!("\nAST live allocations by size class"); + let labels = ["<=8", "<=16", "<=32", "<=64", "<=256", ">256"]; + for (label, n) in labels.iter().zip(ast.buckets) { + println!(" {label:>6} {n:>8}"); + } + + // The number M1 exists to produce: what the real script's tree costs, to + // set against what tests/projection.rs says a lowering of it would weigh. + // + // Larger than traffic-light's own 77968 for the same script because this + // builds rhai with default features. Theirs sets `no_module`, which drops + // a `Namespace` (an inline `StaticVec` plus a hash) from every + // `Expr::Variable` payload — and this script has 457 of them. Compare + // against 77968, not against this, until M6 builds the restricted set. + println!("\nfollow.rhai — {} source bytes", FOLLOW.len()); + println!( + "{:<24} {:>10} {:>10} {:>10}", + "engine.compile (AST)", follow_ast.bytes, follow_ast.count, follow_ast.peak + ); + println!( + "{:<24} {:>10.1}", + "bytes per source byte", + follow_ast.bytes as f64 / FOLLOW.len() as f64 + ); + println!( + "{:<24} {:>10.2}x", + "parser peak / retained", + follow_ast.peak as f64 / follow_ast.bytes as f64 + ); + println!("\nfollow.rhai AST live allocations by size class"); + for (label, n) in labels.iter().zip(follow_ast.buckets) { + println!(" {label:>6} {n:>8}"); + } + + // What loading an artifact costs, against parsing the same script. + // + // This is the claim the byte encoding exists for. `Program::read` borrows + // its instructions from the buffer, so nothing it retains is proportional + // to how long the script is — only to the distinct names and constants it + // mentions. Parsing retains a node per node. + // The measurement the whole project was for: the real script, the real + // tree, and the artifact that replaces it. + let follow_program = Compiler::new().compile(&follow_ast.value); + assert_eq!( + follow_program.residual_count(), + 0, + "follow.rhai must lower completely, or there is nothing to write", + ); + let (follow_artifact, follow_table) = follow_program + .write_stripped() + .expect("follow.rhai must be writable"); + let follow_loaded = + measure(|| rhaigrain::Program::read(&follow_artifact).expect("must load")); + + println!( + "\nfollow.rhai: {} source bytes\n tree {:>8} bytes retained, {:>5} allocs, {:>8} peak\n \ + artifact {:>8} bytes on the wire ({} of debug table kept off it)\n \ + loaded {:>8} bytes retained, {:>5} allocs", + FOLLOW.len(), + follow_ast.bytes, + follow_ast.count, + follow_ast.peak, + follow_artifact.len(), + follow_table.len(), + follow_loaded.bytes, + follow_loaded.count, + ); + println!( + " {:.1}x less retained, and no parser peak at all", + follow_ast.bytes as f64 / follow_loaded.bytes as f64, + ); + + // Measured at several lengths, because the ratio at one length says almost + // nothing. Repeating the same statements grows the instruction stream while + // the set of distinct names stays put — which is exactly the shape the + // claim is about, and the shape a real script has as it gets longer. + println!("\nloading an artifact against parsing the same script"); + println!( + "{:>7} {:>9} {:>9} {:>9} {:>8} {:>7}", + "source", "artifact", "tree", "loaded", "allocs", "ratio" + ); + + let mut ratios = Vec::new(); + for repeats in [1usize, 4, 16] { + let source = LOADABLE.repeat(repeats); + + let tree = measure(|| engine.compile(&source).expect("must compile")); + let program = Compiler::new().compile(&tree.value); + assert_eq!( + program.residual_count(), + 0, + "LOADABLE must lower completely, or this measures the walker", + ); + + let (stripped, _) = program.write_stripped().expect("must be writable"); + let loaded = measure(|| rhaigrain::Program::read(&stripped).expect("must load")); + + // Borrowed, not copied: the code section contributes nothing at all to + // what a load retains, which is the whole reason for the byte encoding. + let code = loaded.value.code(); + assert!( + code.as_ptr() >= stripped.as_ptr() + && code.as_ptr() as usize <= stripped.as_ptr() as usize + stripped.len(), + "the loaded chunk must point into the artifact, not into a copy of it", + ); + + let ratio = tree.bytes as f64 / loaded.bytes as f64; + ratios.push(ratio); + println!( + "{:>7} {:>9} {:>9} {:>9} {:>8} {:>6.1}x", + source.len(), + stripped.len(), + tree.bytes, + loaded.bytes, + loaded.count, + ratio, + ); + } + + // The property, rather than a number that would need updating: what a tree + // retains grows with the program, what a load retains does not. A ratio + // that stopped climbing would mean something in the loader had started + // scaling with length. + assert!( + ratios[2] > ratios[1] && ratios[1] > ratios[0], + "the saving must grow with the program, got {ratios:?}", + ); + + // The instrument has to be working before any milestone can lean on it. + assert!( + ast.count > 0 && ast.bytes > 0, + "the counters saw nothing; the global allocator is not installed", + ); + assert!( + ast.peak >= ast.bytes, + "peak ({}) below retained ({}) means peak tracking is broken", + ast.peak, + ast.bytes, + ); + + // M0's compiler clones the whole statement list into one residual, so the + // Program is a second copy of the tree rather than a saving. Pinning that + // here means M1's first real lowering shows up as this assertion failing, + // which is the intended way to notice progress. + assert!( + program.bytes > 0, + "a whole-program residual should cost about what the tree costs", + ); + + drop(program); + drop(ast); + drop(engine_cost); +} diff --git a/tests/grain/callback.rs b/tests/grain/callback.rs new file mode 100644 index 000000000..4395e31a1 --- /dev/null +++ b/tests/grain/callback.rs @@ -0,0 +1,216 @@ +//! Natives that call a compiled function back. +//! +//! `[1, 2, 3].map(|x| x * 2)` is the shape: `map` is rhai's, and the pointer it +//! is handed is resolved by rhai's dispatch rather than by ours. This is where +//! the wrappers that make that resolve are held to the walker's behaviour, and +//! where the places it still diverges are pinned rather than left to be +//! discovered. + +// Only the engine is wanted here; the corpus scripts belong to the harnesses +// that run all of them. +#[allow(dead_code)] +mod corpus; + +use rhai::{Dynamic, Engine, Scope}; +use rhaigrain::{Compiler, Vm}; + +/// Run a source through the VM with the callback wrappers installed. +fn run(engine: &Engine, source: &str) -> Result { + let ast = engine.compile(source).map_err(|err| format!("{err:?}"))?; + let program = Compiler::new().compile(&ast).into_shared(); + Vm::new(engine) + .run_with_callbacks(&program, &mut Scope::new()) + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")) +} + +/// The same source through the walker, which is what the answer has to be. +fn walk(engine: &Engine, source: &str) -> Result { + let ast = engine.compile(source).map_err(|err| format!("{err:?}"))?; + engine + .eval_ast::(&ast) + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")) +} + +fn agree(source: &str) { + let engine = corpus::engine(); + assert_eq!(walk(&engine, source), run(&engine, source), "{source}"); +} + +/// The array is bound to a variable in every case here, and that is not +/// incidental: a chain rooted at a literal is still a fragment, and a fragment +/// hands the whole expression back to the walker — which would make every one +/// of these pass without the wrappers existing at all. +fn lowered(source: &str) { + let engine = corpus::engine(); + let ast = engine.compile(source).unwrap(); + let program = Compiler::new().compile(&ast); + assert!( + program.residual_count() == 0, + "{source} still fragments, so it does not test the callback path", + ); + assert!(program.makes_fn_pointers(), "{source}"); +} + +#[test] +fn a_native_can_call_a_closure_back() { + lowered("let a = [1, 2, 3]; a.map(|x| x * 2)"); + agree("let a = [1, 2, 3]; a.map(|x| x * 2)"); + agree("let a = [1, 2, 3, 4]; a.filter(|x| x % 2 == 0)"); +} + +#[test] +fn a_native_can_call_a_named_function_back() { + agree("fn double(x) { x * 2 } let a = [1, 2, 3]; a.map(Fn(\"double\"))"); +} + +/// A capture arrives, and has the right value. +/// +/// Multiplication commutes, so this says nothing about which *position* it +/// arrives in — see `a_capturing_closure_reaches_a_native_with_its_arguments_rotated` +/// for that, which is where the answer is unwelcome. +#[test] +#[cfg(not(feature = "no_closure"))] +fn a_capturing_closure_resolves_and_sees_its_capture() { + lowered("let n = 10; let a = [1, 2, 3]; a.map(|x| x * n)"); + agree("let n = 10; let a = [1, 2, 3]; a.map(|x| x * n)"); +} + +/// A capturing closure handed to a native that binds `this` gets its captured +/// values *after* the element instead of before. +/// +/// Not our arithmetic: it is which shape rhai tries first. A capture is a +/// curried value, and `_call_with_extra_args` (`types/fn_ptr.rs:573`) opens +/// with `[this] ++ curry ++ args` — the one order that is never right. Rhai's +/// own closures escape it because they are `Fn*` pointers with the body +/// attached, which is caught two branches earlier at `:538` and rearranged +/// into `curry ++ [this] ++ args`. +/// +/// Ours are `Fn` pointers resolved by name, so rhai reaches them as *natives* +/// and takes the first shape. The proof that this is rhai's behaviour rather +/// than ours is `stock_rhai_does_the_same_to_its_own_native_pointers` below, +/// which reproduces it with no rhaigrain in the picture at all. +/// +/// So: safe for a closure that captures nothing, and for one whose parameters +/// commute. Wrong, silently, otherwise. Non-commutative on purpose here — +/// `n - x` and `x - n` are the same two values in the other order, and only +/// arithmetic that cares can tell them apart. +#[test] +#[cfg(not(feature = "no_closure"))] +fn a_capturing_closure_reaches_a_native_with_its_arguments_rotated() { + let engine = corpus::engine(); + let source = "let n = 10; let a = [1, 2, 3]; a.map(|x| n - x)"; + + assert_eq!(walk(&engine, source), Ok("[9, 8, 7]".to_string())); + assert_eq!(run(&engine, source), Ok("[-9, -8, -7]".to_string())); +} + +/// The same rotation, with nothing of ours involved. +/// +/// `Fn(s)` on a computed name is the one spelling that gets a name-only +/// pointer out of stock rhai, which is what every pointer we make is. Curry it, +/// point it at a native, and hand it to `map`: rhai passes the element first +/// and the curried value second, while `f.call(1)` on the very same pointer +/// passes them the other way round. +#[test] +fn stock_rhai_does_the_same_to_its_own_native_pointers() { + let mut engine = corpus::engine(); + engine.register_fn("nsub", |a: i64, b: i64| a - b); + + let curried = "let s = \"ns\" + \"ub\"; let f = Fn(s).curry(10);"; + assert_eq!( + walk(&engine, &format!("{curried} [1, 2, 3].map(f)")), + Ok("[-9, -8, -7]".to_string()), + "rhai puts the element before the curried value", + ); + assert_eq!( + walk(&engine, &format!("{curried} f.call(1)")), + Ok("9".to_string()), + "and the curried value first when there is no element", + ); +} + +#[test] +fn a_callback_reaching_a_second_one_still_resolves() { + // The limit is raised because reaching a chunk through rhai's dispatch + // costs more call levels than reaching a script function does, and the + // default in a debug build is 8. `a_callback_costs_more_call_levels` + // measures the difference; this is about resolution, not depth. + let mut engine = corpus::engine(); + engine.set_max_call_levels(64); + let source = "let a = [[1, 2], [3, 4]]; a.map(|row| row.map(|x| x + 1))"; + assert_eq!(walk(&engine, source), run(&engine, source)); +} + +#[test] +#[cfg(not(feature = "no_closure"))] +fn a_closure_called_back_sees_a_write_made_after_it_was_made() { + agree("let n = 1; let f = |x| x * n; n = 10; let a = [1, 2, 3]; a.map(f)"); +} + +#[test] +fn an_error_inside_a_callback_still_arrives() { + let engine = corpus::engine(); + let err = run(&engine, "let a = [1, 2, 3]; a.map(|x| throw x)").unwrap_err(); + assert!(err.contains("ErrorInFunctionCall"), "{err}"); +} + +/// Reaching a chunk through rhai's dispatch spends more of the call budget +/// than reaching a script function does, so a callback nests less deeply before +/// `max_call_levels` stops it. +/// +/// Measured rather than asserted at a number, because the number is a property +/// of rhai's dispatch and would drift. What has to hold is the direction: we +/// are stricter, never laxer. A program that would have run out of budget on +/// the walker must not somehow keep going here. +#[test] +fn a_callback_costs_more_call_levels() { + let deepest = |depth: usize, run: &dyn Fn(&Engine, &str) -> Result| { + // `depth` closures nested through `map`, each one a boundary, over an + // array nested to match. + let mut source = format!("let a = {}1{}; a", "[".repeat(depth), "]".repeat(depth)); + for level in 0..depth { + source.push_str(&format!(".map(|v{level}| v{level}")); + } + source.push_str(" + 1"); + source.push_str(&")".repeat(depth)); + (1..=64) + .find(|levels| { + let mut engine = corpus::engine(); + engine.set_max_call_levels(*levels); + run(&engine, &source).is_ok() + }) + .unwrap_or(usize::MAX) + }; + + for depth in 1..=3 { + let walker = deepest(depth, &walk); + let vm = deepest(depth, &run); + println!("{depth} nested callback(s): walker needs {walker}, we need {vm}"); + assert!( + vm >= walker, + "depth {depth}: we ran at {vm} levels where the walker needed {walker}, \ + so a budget the walker respects is not being enforced", + ); + } +} + +#[test] +fn without_the_wrappers_the_pointer_does_not_resolve() { + // The whole reason `run_with_callbacks` exists. A plain `run` leaves rhai + // nowhere to look, and the failure is a lookup failure rather than + // anything worse. + let engine = corpus::engine(); + let source = "let a = [1, 2, 3]; a.map(|x| x * 2)"; + let ast = engine.compile(source).unwrap(); + let program = Compiler::new().compile(&ast); + + let err = Vm::new(&engine) + .run(&program, &mut Scope::new()) + .unwrap_err(); + assert!( + format!("{err:?}").contains("ErrorFunctionNotFound"), + "{err}" + ); +} diff --git a/tests/grain/corpus/generate.rs b/tests/grain/corpus/generate.rs new file mode 100644 index 000000000..01b60b715 --- /dev/null +++ b/tests/grain/corpus/generate.rs @@ -0,0 +1,562 @@ +// Random rhai scripts, for comparing the VM against the walker on programs +// nobody wrote. +// +// The hand-written corpus proves the constructs someone thought to test. This +// covers the combinations they did not: a `try` inside a `for` inside a +// `switch` arm whose subject is an indexed chain, and the several thousand +// other shapes of that kind. Every one is run both ways and the results +// compared, so a divergence names a script that reproduces it. +// +// Two constraints follow from `fuzz/fuzz_targets/generated.rs` pulling this +// file in with `include!` rather than depending on it: nothing may be imported +// beyond `std`, and the header is `//` rather than `//!`, because an included +// file cannot open with an inner attribute. +// +// What is deliberately not generated: +// +// - Recursion. A function may only call one defined before it, so no script +// can recurse. Running out of call levels is a limit both sides enforce +// but not in lockstep, and it would drown real findings. +// - A closure handed to a native. `a.map(|x| ..)` has a known divergence in +// argument order for captures, pinned in `tests/callback.rs`. Closures are +// generated and called directly instead. +// - An unbounded loop. Every `while` counts a guard variable the body cannot +// name, so it always terminates. + +/// A source of choices: xorshift64*, optionally front-loaded with bytes +/// somebody else picked. +/// +/// The bytes are for `cargo fuzz`. Every grammar decision takes one, so +/// flipping byte *k* of an input changes decision *k* and leaves the rest of +/// the script alone — which is the locality a coverage-guided fuzzer needs to +/// make progress. From a plain seed, or once the bytes run out, it is an +/// ordinary reproducible PRNG. +pub struct Rng { + state: u64, + choices: Vec, + at: usize, +} + +impl Rng { + #[must_use] + pub fn new(seed: u64) -> Self { + Self { + // A zero state is a fixed point for xorshift, and zero is exactly + // the seed somebody reaches for first. + state: if seed == 0 { 0x9e37_79b9_7f4a_7c15 } else { seed }, + choices: Vec::new(), + at: 0, + } + } + + /// Draws from `choices` until they are used up, then from the PRNG. + /// + /// Running out is not a failure and not a truncation: a script that stops + /// being fuzzer-directed part way through is still a valid script, and + /// still worth comparing. + #[must_use] + pub fn from_bytes(choices: &[u8]) -> Self { + let mut seed = 0u64; + for byte in choices.iter().take(8) { + seed = (seed << 8) | u64::from(*byte); + } + Self { + choices: choices.to_vec(), + at: 0, + ..Self::new(seed) + } + } + + pub fn next(&mut self) -> u64 { + let mut x = self.state; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.state = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + pub fn below(&mut self, bound: usize) -> usize { + if bound == 0 { + return 0; + } + if let Some(byte) = self.choices.get(self.at) { + self.at += 1; + return usize::from(*byte) % bound; + } + (self.next() % bound as u64) as usize + } + + fn one_of<'a, T>(&mut self, choices: &'a [T]) -> &'a T { + &choices[self.below(choices.len())] + } + + fn chance(&mut self, n: usize) -> bool { + self.below(n) == 0 + } +} + +/// How deep an expression may nest. +/// +/// Rhai's parser has its own limit and rejects anything past it. Staying well +/// under means a rejected parse is a bug in this generator rather than the +/// expected outcome for half the corpus. +const MAX_DEPTH: usize = 3; + +/// How deep blocks and control flow may nest, for the same reason. +/// +/// Rhai counts statements towards the same budget and allows a function body +/// half of what it allows the top level, so this is what keeps a script inside +/// both. Scripts are wide rather than deep as a result, which is the right +/// trade: a `try` inside a `for` finds things, a `try` inside a `for` inside a +/// `while` inside an `if` mostly finds the parser. +const MAX_NESTING: usize = 2; + +/// Operators whose *result* is a boolean whatever their operands turn out to +/// be, so the parser accepts one as a condition. +/// +/// It type-checks that much statically: `if 5 { }` is a parse error, not a +/// runtime one. Comparisons are safe to build out of anything because only the +/// operands can be wrong, and that is a runtime disagreement worth generating. +const COMPARISONS: &[&str] = &["==", "!=", "<", "<=", ">", ">="]; + +/// Operators that are defined for at least one pair of generated types. A pair +/// they are not defined for is a runtime error both sides have to agree on, +/// which is worth generating rather than worth avoiding. +/// +/// `&&` and `||` are not here — they demand boolean operands at parse time, so +/// they belong to [`Generator::condition`] and nowhere else. +const BINARY: &[&str] = &[ + "+", "-", "*", "/", "%", "==", "!=", "<", "<=", ">", ">=", "&", "|", +]; + +const METHODS: &[&str] = &["len", "to_string", "abs", "is_empty", "floor", "to_upper"]; + +/// Natives called in *function-call* style, with their arity. +/// +/// Rhai rewrites `f(x, ..)` into `x.f(..)` when the first argument is a plain +/// variable, so `push(a, 2)` mutates `a` and `push([1], 2)` mutates nothing — +/// the same syntax, and only the receiver decides. Half of these mutate and +/// half do not, because whether the write lands is the question. +/// +/// Nothing here can grow a value without bound: an argument reaching one of +/// these is an arbitrary generated integer, and `pad` would turn that into an +/// allocation rather than into a divergence. +const NATIVES: &[(&str, usize)] = &[ + ("push", 2), + ("insert", 3), + ("remove", 2), + ("truncate", 2), + ("reverse", 1), + ("clear", 1), + ("pop", 1), + ("shift", 1), + ("len", 1), + ("is_empty", 1), + ("abs", 1), + ("to_upper", 1), +]; + +pub struct Generator { + rng: Rng, + /// Variables that can be named, innermost last. + vars: Vec, + /// Functions defined so far, with their arity. Only these can be called, + /// and only from after their definition, which is what rules out recursion. + functions: Vec<(String, usize)>, + /// Loop nesting, so `break` and `continue` appear only where they parse. + loops: usize, + depth: usize, + /// Nesting of blocks and control flow, capped for the same reason `depth` + /// is: rhai counts statements towards the same complexity limit, so a + /// `while` inside an `if` inside a `for` runs out of budget on its own. + nesting: usize, + /// Set while generating the inside of an interpolated string, where + /// another one cannot go — the lexer ends the outer string on the inner + /// one's opening backtick. + interpolating: bool, + /// Supplies names that cannot collide with each other. + counter: usize, +} + +impl Generator { + #[must_use] + pub fn new(seed: u64) -> Self { + Self { + rng: Rng::new(seed), + vars: Vec::new(), + functions: Vec::new(), + loops: 0, + depth: 0, + nesting: 0, + interpolating: false, + counter: 0, + } + } + + /// A generator whose choices are made by somebody else's bytes, for + /// `cargo fuzz`. See [`Rng::from_bytes`]. + #[must_use] + pub fn from_bytes(choices: &[u8]) -> Self { + Self { + rng: Rng::from_bytes(choices), + ..Self::new(0) + } + } + + fn name(&mut self, prefix: &str) -> String { + self.counter += 1; + format!("{prefix}{}", self.counter) + } + + /// One whole script. + pub fn script(&mut self) -> String { + let mut out = String::new(); + + for _ in 0..self.rng.below(3) { + out.push_str(&self.function()); + out.push(' '); + } + + // A scope of its own per script, so nothing leaks between them. + self.vars.clear(); + for _ in 0..1 + self.rng.below(5) { + out.push_str(&self.statement()); + out.push(' '); + } + + // Ending on an expression gives the script a value, so a divergence in + // what it computed shows up and not only a divergence in what it left + // behind. + out.push_str(&self.expression()); + out + } + + fn function(&mut self) -> String { + let name = self.name("f"); + let arity = self.rng.below(3); + let params: Vec = (0..arity).map(|i| format!("p{i}")).collect(); + + // A body sees its parameters and nothing else, which is also what rhai + // gives it. + let outer = std::mem::replace(&mut self.vars, params.clone()); + let outer_loops = std::mem::replace(&mut self.loops, 0); + // Rhai allows a function body half the expression depth it allows the + // top level (`MAX_FUNCTION_EXPR_DEPTH`), so a body starts one level in. + let outer_nesting = std::mem::replace(&mut self.nesting, 1); + let mut body = String::new(); + for _ in 0..1 + self.rng.below(3) { + body.push_str(&self.statement()); + body.push(' '); + } + body.push_str(&self.expression()); + self.vars = outer; + self.loops = outer_loops; + self.nesting = outer_nesting; + + // Registered after the body is generated, so the body cannot call it. + self.functions.push((name.clone(), arity)); + format!("fn {name}({}) {{ {body} }}", params.join(", ")) + } + + fn statement(&mut self) -> String { + // Weighted by hand: `let` more often than anything else, because a + // script with no variables cannot exercise slots, chains or capture. + // Anything that nests is dropped from the choices once deep enough, + // which is why the arms are ordered with those last. + let nesting_kinds = if self.nesting < MAX_NESTING { 5 } else { 0 }; + let loop_kinds = if self.loops > 0 { 2 } else { 0 }; + match self.rng.below(5 + nesting_kinds + loop_kinds) { + 0..=2 => { + let value = self.expression(); + let name = self.name("v"); + self.vars.push(name.clone()); + format!("let {name} = {value};") + } + 3 => self.assignment(), + 4 => format!("{};", self.expression()), + + // Nesting, reachable only while there is budget for it. + 5 => { + let cond = self.condition(); + self.nesting += 1; + let then = self.block(); + let otherwise = self.rng.chance(2).then(|| self.block()); + self.nesting -= 1; + match otherwise { + Some(otherwise) => format!("if {cond} {then} else {otherwise}"), + None => format!("if {cond} {then}"), + } + } + 6 => self.nested(Self::bounded_while), + 7 => self.nested(Self::for_loop), + 8 => self.nested(Self::switch), + 9 => self.nested(Self::try_catch), + + // Only reachable inside a loop, per `loop_kinds` above. + 10 => "break;".to_string(), + _ => "continue;".to_string(), + } + } + + fn nested(&mut self, build: fn(&mut Self) -> String) -> String { + self.nesting += 1; + let out = build(self); + self.nesting -= 1; + out + } + + /// A run of statements, with the scope they declared popped afterwards + /// exactly as the VM pops it. + /// + /// Returned without braces so a caller can append to it. Appending to a + /// finished block by trimming its `}` back off is how a body that happens + /// to end in one gets silently truncated. + fn statements(&mut self) -> String { + let mark = self.vars.len(); + let mut out = String::new(); + for _ in 0..1 + self.rng.below(2) { + out.push_str(&self.statement()); + out.push(' '); + } + self.vars.truncate(mark); + out + } + + fn block(&mut self) -> String { + format!("{{ {} }}", self.statements()) + } + + fn assignment(&mut self) -> String { + let Some(target) = self.variable() else { + return format!("{};", self.expression()); + }; + let op = self.rng.one_of(&["=", "+=", "-=", "*="]).to_string(); + format!("{target} {op} {};", self.expression()) + } + + /// `while`, with a guard the body cannot reach. + /// + /// The guard is not pushed onto `vars`, so nothing generated inside can + /// assign to it and the loop always ends. + fn bounded_while(&mut self) -> String { + let guard = self.name("g"); + let limit = 1 + self.rng.below(4); + self.loops += 1; + let body = self.statements(); + self.loops -= 1; + format!("let {guard} = 0; while {guard} < {limit} {{ {body} {guard} += 1; }}") + } + + fn for_loop(&mut self) -> String { + let item = self.name("it"); + let iterable = if self.rng.chance(2) { + format!("0..{}", 1 + self.rng.below(4)) + } else { + self.array() + }; + + self.vars.push(item.clone()); + self.loops += 1; + let body = self.block(); + self.loops -= 1; + self.vars.pop(); + + format!("for {item} in {iterable} {body}") + } + + fn switch(&mut self) -> String { + let subject = self.expression(); + let mut arms = Vec::new(); + for value in 0..1 + self.rng.below(3) { + arms.push(format!("{value} => {}", self.expression())); + } + if self.rng.chance(3) { + arms.push(format!("0..={} => {}", 9, self.expression())); + } + arms.push(format!("_ => {}", self.expression())); + format!("switch {subject} {{ {} }};", arms.join(", ")) + } + + fn try_catch(&mut self) -> String { + let mut body = self.statements(); + // A `throw` inside, most of the time: rhai's optimizer replaces a `try` + // whose body is pure with a plain block, so a pure body would test the + // optimizer rather than the handler. + if !self.rng.chance(3) { + let thrown = self.expression(); + body.push_str(&format!("throw {thrown};")); + } + let catch = self.block(); + if self.rng.chance(2) { + format!("try {{ {body} }} catch (e) {catch}") + } else { + format!("try {{ {body} }} catch {catch}") + } + } + + fn variable(&mut self) -> Option { + if self.vars.is_empty() { + return None; + } + Some(self.vars[self.rng.below(self.vars.len())].clone()) + } + + /// An array literal, empty about a quarter of the time. + /// + /// Empty matters on its own: it contributes no size check, so nesting one + /// inside a literal that does is what catches a running total being popped + /// by the wrong literal. + fn array(&mut self) -> String { + let items: Vec = (0..self.rng.below(4)).map(|_| self.element()).collect(); + format!("[{}]", items.join(", ")) + } + + fn map(&mut self) -> String { + let entries: Vec = (0..self.rng.below(4)) + .map(|n| format!("k{n}: {}", self.element())) + .collect(); + format!("#{{ {} }}", entries.join(", ")) + } + + /// What goes inside a literal. + /// + /// Sometimes another literal, so nesting is reached — an all-constant one + /// is folded away by the optimizer before the compiler sees it, and only a + /// computed element keeps the literal alive to run time. + fn element(&mut self) -> String { + if self.depth < MAX_DEPTH && self.rng.chance(4) { + self.depth += 1; + let out = if self.rng.chance(2) { + self.array() + } else { + self.map() + }; + self.depth -= 1; + return out; + } + self.atom() + } + + /// A leaf: cheap, and always available however deep the nesting has got. + fn atom(&mut self) -> String { + match self.rng.below(8) { + 0..=2 => format!("{}", self.rng.below(64)), + 3 => format!("{}.{}", self.rng.below(8), self.rng.below(8)), + 4 => format!("\"s{}\"", self.rng.below(8)), + 5 => self.rng.one_of(&["true", "false"]).to_string(), + 6 => "'c'".to_string(), + _ => self.variable().unwrap_or_else(|| "1".to_string()), + } + } + + /// Something the parser will accept where a boolean is required. + fn condition(&mut self) -> String { + if self.depth >= MAX_DEPTH { + return self.rng.one_of(&["true", "false"]).to_string(); + } + self.depth += 1; + let out = match self.rng.below(8) { + 0 => "true".to_string(), + 1 => "false".to_string(), + 2 => format!("(!{})", self.condition()), + 3 => { + let lhs = self.condition(); + let op = self.rng.one_of(&["&&", "||"]); + format!("({lhs} {op} {})", self.condition()) + } + _ => { + let lhs = self.expression(); + let op = self.rng.one_of(COMPARISONS); + format!("({lhs} {op} {})", self.expression()) + } + }; + self.depth -= 1; + out + } + + pub fn expression(&mut self) -> String { + if self.depth >= MAX_DEPTH { + return self.atom(); + } + self.depth += 1; + let out = self.compound(); + self.depth -= 1; + out + } + + fn compound(&mut self) -> String { + match self.rng.below(14) { + 0..=3 => self.atom(), + 4 | 5 => { + let lhs = self.expression(); + let op = self.rng.one_of(BINARY); + format!("({lhs} {op} {})", self.expression()) + } + 6 => format!("(-{})", self.expression()), + // `!` takes a condition rather than any expression: rhai rejects + // `!9` at parse time the same way it rejects `if 9`. + 7 => format!("(!{})", self.condition()), + 8 => self.array(), + 9 => self.map(), + 10 => { + // A chain over each of the three `Root` variants: a declared + // variable, a name that is declared nowhere — which is what a + // caller would have supplied, and here resolves to nothing — + // and a literal. + let root = match self.variable() { + _ if self.rng.chance(6) => "absent".to_string(), + Some(var) if self.rng.chance(2) => var, + _ => self.array(), + }; + match self.rng.below(3) { + 0 => format!("{root}[{}]", self.rng.below(4)), + 1 => format!("{root}.{}()", self.rng.one_of(METHODS)), + _ => format!("{root}.a"), + } + } + 11 if !self.interpolating => { + self.interpolating = true; + let segment = self.expression(); + self.interpolating = false; + format!("`i{}{{{segment}}}`", self.rng.below(8)) + } + 11 => self.atom(), + 12 => self.call(), + _ => { + // A closure, called directly. Handing one to a native is the + // shape with the known divergence, so it is not generated. + let param = self.name("c"); + self.vars.push(param.clone()); + let body = self.expression(); + self.vars.pop(); + format!("(|{param}| {body}).call({})", self.atom()) + } + } + } + + fn call(&mut self) -> String { + if self.rng.chance(4) { + return self.native_call(); + } + if self.functions.is_empty() { + return self.atom(); + } + let (name, arity) = self.functions[self.rng.below(self.functions.len())].clone(); + let args: Vec = (0..arity).map(|_| self.atom()).collect(); + format!("{name}({})", args.join(", ")) + } + + /// A native in function-call style, with a variable in first position + /// wherever there is one to use — see [`NATIVES`]. + fn native_call(&mut self) -> String { + let (name, arity) = NATIVES[self.rng.below(NATIVES.len())]; + let receiver = self.variable().unwrap_or_else(|| self.array()); + let rest: Vec = (1..arity).map(|_| self.atom()).collect(); + + if rest.is_empty() { + return format!("{name}({receiver})"); + } + format!("{name}({receiver}, {})", rest.join(", ")) + } +} diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs new file mode 100644 index 000000000..0336450a0 --- /dev/null +++ b/tests/grain/corpus/mod.rs @@ -0,0 +1,663 @@ +//! Scripts the VM must agree with rhai on. +//! +//! Weighted towards the places a bytecode VM is most likely to drift from a +//! tree walker rather than towards breadth: scope discipline, the error-based +//! unwinding rhai uses for `return`/`break`/`throw`, and the lvalue forms that +//! cannot be expressed as a plain `&mut` and so need explicit write-back. +//! +//! Everything here currently runs as a single `EvalAst` residual, so passing is +//! expected. That is the point — it pins the baseline, and it exercises the +//! runtime-state setup (function library, module resolver, source name, +//! `return`/`exit` mapping) which is real code that can be wrong today. + +// Only `tests/fuzz.rs` and the `generated` fuzz target use this; the other +// harnesses take the module for its cases. +#[allow(dead_code)] +pub mod generate; + +pub struct Case { + pub name: &'static str, + pub source: &'static str, +} + +/// A host type with a getter, a setter, an indexer and both kinds of method. +/// +/// Registered so the corpus can reach the one part of the chain walker that is +/// conservative rather than exact. Arrays and maps hand out references, so a +/// mutation partway down a chain lands in them and nothing needs writing back. +/// A getter hands back a *value*, so `w.inner.level = 1` mutates a temporary +/// and the setter is the only way home — and rhai decides whether to call it +/// from `func.is_method()`, which the VM cannot see and therefore approximates. +/// Without a host type in the engine, nothing here is ever exercised. +#[derive(Debug, Clone, Default)] +pub struct Widget { + pub level: i64, + pub cells: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct Holder { + pub inner: Widget, +} + +fn out_of_range(index: i64, len: usize) -> Box { + Box::new(rhai::EvalAltResult::ErrorArrayBounds( + len, + index, + rhai::Position::NONE, + )) +} + +/// The engine both sides of the differential run against. +pub fn engine() -> rhai::Engine { + let mut engine = rhai::Engine::new(); + + engine + .register_type_with_name::("Widget") + .register_fn("widget", |level: i64| Widget { + level, + cells: vec![10, 20, 30], + }) + .register_get_set( + "level", + |w: &mut Widget| w.level, + |w: &mut Widget, v: i64| w.level = v, + ) + // Returning an error rather than panicking, because a panic in a + // registered function takes the test process with it. + .register_indexer_get_set( + |w: &mut Widget, i: i64| -> Result> { + w.cells + .get(i as usize) + .copied() + .ok_or_else(|| out_of_range(i, w.cells.len())) + }, + |w: &mut Widget, i: i64, v: i64| -> Result<(), Box> { + let len = w.cells.len(); + *w.cells + .get_mut(i as usize) + .ok_or_else(|| out_of_range(i, len))? = v; + Ok(()) + }, + ) + // Takes the receiver by reference, so rhai counts it as a method and + // writes a temporary back afterwards. + .register_fn("bump", |w: &mut Widget| w.level += 1) + // Reads only. Whether rhai still writes back after one of these is + // exactly the question this corpus is here to settle. + .register_fn("doubled", |w: &mut Widget| w.level * 2); + + engine + .register_type_with_name::("Holder") + .register_fn("holder", |level: i64| Holder { + inner: Widget { + level, + cells: vec![1, 2, 3], + }, + }) + .register_get_set( + "inner", + |h: &mut Holder| h.inner.clone(), + |h: &mut Holder, w: Widget| h.inner = w, + ); + + engine +} + +const fn case(name: &'static str, source: &'static str) -> Case { + Case { name, source } +} + +pub const CASES: &[Case] = &[ + // --- values and operators ------------------------------------------- + case("int_arithmetic", "let a = 7; let b = 3; a * b - a / b + a % b"), + case("float_arithmetic", "let a = 7.5; let b = 0.5; a * b + a / b"), + case("mixed_numeric", "1 + 2.5"), + case("comparison_chain", "let a = 5; a > 1 && a < 10 || a == 5"), + case("bitwise", "let a = 0b1010; (a & 0b0110) | (a ^ 0b1111) << 2"), + case("string_ops", r#"let s = "hello"; s + " " + "world" + s.len"#), + case("string_interpolation", r#"let n = 42; `answer is ${n} and ${n * 2}`"#), + // Every segment type goes through a different arm of rhai's rendering: + // strings skip dispatch entirely, unit renders empty, and a container + // gets its debug-ish form. + case( + "interpolation_of_every_type", + r#"let s = "x"; let n = 1; let f = 1.5; let b = true; let u = (); `${s}|${n}|${f}|${b}|${u}|`"#, + ), + case( + "interpolation_of_containers", + r#"let a = [1, 2]; let m = #{ k: 1 }; `${a}|${m}`"#, + ), + // A host type with no `to_string` registered falls back to the mapped + // type name rather than to `Debug`. + case("interpolation_of_host_type", r#"let w = widget(3); `w=${w}`"#), + case("char_ops", r#"let c = 'a'; c.to_upper()"#), + case("unit_value", "()"), + + // --- containers ------------------------------------------------------- + case("array_literal", "let a = [1, 2, 3]; a[0] + a[1] + a[2]"), + case("array_methods", "let a = [3, 1, 2]; a.sort(); a"), + case("map_literal", r#"let m = #{ a: 1, b: 2 }; m.a + m.b"#), + // The one above is all-constant, so the optimizer folds it and the map + // never gets built at run time. These do get built: rhai keeps a template + // holding every key and fills the computed ones in afterwards. + case("map_computed_value", "let v = 7; let m = #{ a: v, b: 2 }; m.a + m.b"), + case("map_all_computed", "let v = 7; let w = 8; #{ a: v, b: w }"), + case("map_computed_nested", "let v = 7; #{ outer: #{ inner: v } }.outer.inner"), + case("map_computed_in_array", "let v = 7; [#{ a: v }, #{ a: 2 }]"), + // An empty literal inside one that is not. It contributes no size check of + // its own, so it must not consume the enclosing literal's running total. + case("empty_literals_nested_in_computed_ones", "let v = 7; [v, [], #{}, v]"), + case("empty_map_nested_in_a_computed_map", "let v = 7; #{ a: v, b: #{}, c: [] }"), + // The value is a call, so the order it runs in relative to the rest of the + // literal is observable. + case( + "map_computed_order", + r#"let log = ""; fn note(s, c) { s + c } let m = #{ a: note("", "x"), b: note("", "y") }; m.a + m.b"#, + ), + case("nested_containers", r#"let m = #{ xs: [1, 2, #{ y: 3 }] }; m.xs[2].y"#), + + // --- control flow ----------------------------------------------------- + case("if_else", "let a = 5; if a > 3 { \"big\" } else { \"small\" }"), + case("while_loop", "let i = 0; let s = 0; while i < 5 { s += i; i += 1; } s"), + case("do_while", "let i = 0; do { i += 1; } while i < 3; i"), + case("do_until", "let i = 0; do { i += 1; } until i >= 3; i"), + case("loop_break_value", "let i = 0; loop { i += 1; if i > 4 { break i * 10; } }"), + case("continue_skips", "let s = 0; for i in 0..10 { if i % 2 == 0 { continue; } s += i; } s"), + case("for_range", "let s = 0; for i in 0..5 { s += i; } s"), + case("for_array", "let s = 0; for x in [10, 20, 30] { s += x; } s"), + case( + "for_with_counter", + "let s = 0; for (x, i) in [10, 20, 30] { s += x * i; } s", + ), + // The loop variable is pushed once and mutated in place rather than + // re-pushed each iteration (eval/stmt.rs:752); a VM that re-pushes would + // leave the scope a different depth. + case( + "for_loop_var_not_leaked", + "let x = 99; for x in 0..3 { } x", + ), + case("nested_loops_break", "let s = 0; for i in 0..3 { for j in 0..3 { if j == 2 { break; } s += 1; } } s"), + // An empty body is a separate path in rhai that never touches the loop + // variable or the counter (`eval/stmt.rs:719`). + case("for_empty_body", "let s = 0; for i in 0..5 { } s"), + // `return` out of a `for` skips the exhaustion path, so the iterator and + // both loop variables have to go with the frame. + case( + "for_return_from_body", + "fn find(xs) { for (x, i) in xs { if x > 1 { return i; } } -1 } find([1, 2, 3])", + ), + // A `break` out of a `while` nested in a `for` must drop nothing, and out + // of the `for` must drop one — the two are easy to get the wrong way round. + case( + "for_around_while_break", + "let s = 0; for i in 0..3 { let j = 0; while true { j += 1; if j > 2 { break; } s += 1; } } s", + ), + // Iterating a shared cell walks a snapshot, because rhai flattens the + // iterable before asking for an iterator (`eval/stmt.rs:677`). + case( + "for_over_captured_array", + "let a = [1, 2, 3]; { let f = || a; } let s = 0; for x in a { s += x; } s", + ), + + // --- switch ----------------------------------------------------------- + case("switch_literal", "let x = 2; switch x { 1 => \"one\", 2 => \"two\", _ => \"other\" }"), + case("switch_range", "let x = 42; switch x { 0..=9 => \"small\", 10..=99 => \"medium\", _ => \"large\" }"), + // A failing guard must fall through to the next matching case, not to the + // default, so both single-digit arms are needed to tell those apart. + case( + "switch_guard", + "let x = 5; switch x { 0..=9 if x % 2 == 1 => \"odd digit\", 0..=9 => \"even digit\", _ => \"big\" }", + ), + case("switch_default_only", "switch 999 { 1 => \"a\", _ => \"fallback\" }"), + // Two case values, one arm: the table has two entries pointing at one + // body, which a compiler emitting a body per entry would duplicate. + case( + "switch_shared_body", + "let x = 2; switch x { 1 | 2 => \"low\", 3 => \"three\", _ => \"other\" }", + ), + // The rule that reads like a bug and is not: a case value that matched but + // whose guard declined goes to the *default*, never on to the ranges + // (eval/stmt.rs:544). Without the range arm here the two are the same + // answer and the case proves nothing. + case( + "switch_declined_case_skips_ranges", + "let f = false; let x = 1; switch x { 1 if f => \"guarded\", 0..=5 => \"range\", _ => \"default\" }", + ), + // No `_` arm at all, so the miss has to produce unit from nowhere. + case("switch_no_default", "let x = 9; switch x { 1 => \"a\" }"), + case( + "switch_string", + "let s = \"b\"; switch s { \"a\" => 1, \"b\" => 2, _ => 0 }", + ), + // A range arm covers the reals between its bounds, so a float lands in one + // even though the bounds are integers. + case( + "switch_float_in_range", + "let x = 5.5; switch x { 0..10 => \"in\", _ => \"out\" }", + ), + // Hashing a host type panics, so the subject has to be checked before it + // reaches a hasher — and must still find the default. + case( + "switch_unhashable_subject", + "let w = widget(3); switch w { 1 => \"int\", _ => \"other\" }", + ), + // A shared value is not hashable either, so rhai skips the cases *and* the + // ranges and goes straight to the default — however well the value would + // otherwise have matched. Reading the subject through its cell would hide + // that, which is why the subject is loaded unflattened. + case( + "switch_on_a_shared_subject_takes_the_default", + r#"let v = 0; { let f = || v; } switch v { 0 => "case", _ => "default" }"#, + ), + case( + "switch_range_on_a_shared_subject_takes_the_default", + r#"let v = 5; { let f = || v; } switch v { 0..=9 => "range", _ => "default" }"#, + ), + // The same shape before anything shares it, so the pair says the difference + // is the sharing rather than the switch. + case( + "switch_on_an_unshared_subject_matches", + r#"let v = 0; switch v { 0 => "case", _ => "default" }"#, + ), + // An arm body is a block: it declares, and it has to leave the scope the + // depth it found it — which only shows up in something that reads a local + // afterwards. + case( + "switch_block_body_scope", + "let x = 1; let y = 0; switch x { 1 => { let z = 5; y = z * 2 }, _ => () } y", + ), + // A jump out of an arm and out of the switch, which is where the operand + // stack most plausibly ends up a different depth on the two paths. + // A range is a host type as far as `Dynamic` is concerned, and + // `is_hashable` says no to those — even though `Hash for Dynamic` would in + // fact hash a range (types/dynamic.rs:465). So rhai never matches a range + // *subject* against anything, and neither may the VM: mirroring the gate + // matters more than being clever about it. + case( + "switch_range_subject_never_matches", + "let r = 0..5; switch r { 0..5 => \"same\", _ => \"no\" }", + ), + case( + "switch_break_from_loop", + "let s = 0; let i = 0; while i < 10 { switch i { 3 => break, _ => () } s += 1; i += 1; } s", + ), + + // --- blocks used for their value --------------------------------------- + // Rhai wraps a block in `Expr::Stmt` wherever a value is wanted, so these + // are one construct in three disguises. Each declares inside the block, so + // a lowering that forgot to rewind would leave the scope a different depth + // and every slot after it would name the wrong variable. + case( + "let_from_switch", + "let x = 2; let y = switch x { 1 => \"one\", 2 => \"two\", _ => \"other\" }; y", + ), + case( + "let_from_if", + "let c = true; let y = if c { let a = 1; a } else { let b = 2; b }; y + 10", + ), + case("let_from_block", "let a = 3; let y = { let z = a; z * 2 }; y"), + // A block among a call's arguments, where the scope grows while operands + // are already on the stack. + case( + "block_as_argument", + "fn add(a, b) { a + b } let n = 2; add({ let t = n; t + 1 }, 10)", + ), + + // --- scoping ---------------------------------------------------------- + case("shadowing_nested", "let x = 1; { let x = 2; { let x = 3; } } x"), + case("block_scope_discarded", "let x = 1; { let y = 2; x += y; } x"), + case("const_read", "const K = 10; K * 2"), + + // --- functions -------------------------------------------------------- + case("fn_call", "fn add(a, b) { a + b } add(2, 3)"), + // Kept shallow deliberately: rhai's default call-depth limit is far lower + // in debug builds than in release, and this case is about recursion working + // at all, not about the limit. The limit gets its own case. + case("fn_recursion", "fn fib(n) { if n < 2 { n } else { fib(n - 1) + fib(n - 2) } } fib(6)"), + case("fn_early_return", "fn f(x) { if x > 0 { return \"pos\"; } \"nonpos\" } f(1) + f(-1)"), + // A script method mutating its receiver: `this` is bound by reference, so + // the write has to land back in the caller's variable. + case("fn_mutating_method", "fn double() { this *= 2; } let v = 21; v.double(); v"), + case("top_level_return", "let x = 5; if x > 0 { return x * 2; } 0"), + // A script function this compiler could not lower still runs — rhai finds + // it in `global.lib` — and it must run in a scope of its own. Handing it + // this frame's would let the body read the caller's locals, where rhai + // gives it an empty one (`func/call.rs:1476`), and the read is the whole + // difference: the walker cannot find `secret` and a VM that leaked its + // scope answers with 42. + // + // `this` is what leaves the body unlowerable, and calling it by name + // rather than as a method is what routes it through generic dispatch. + case( + "error_a_skipped_function_cannot_see_the_caller", + "fn peek(k) { let seen = secret; this } let secret = 42; peek(1)", + ), + + // --- a variable in first-argument position ------------------------------ + // Rhai rewrites `f(x, ..)` into `x.f(..)` so a `&mut` first parameter + // mutates the variable (`func/call.rs:1434`). These are the same calls the + // chain cases make in method syntax, and they have to mean the same thing. + case("call_style_mutating_native", "let a = [1]; push(a, 2); a"), + case("call_style_mutating_host_type", "let w = widget(4); bump(w); w.level"), + case("call_style_pure_native", "let a = [1, 2]; len(a)"), + // Rhai reads the variable *after* the other arguments, so an argument that + // writes to it is seen. Two shapes of write, because one goes through the + // rewrite itself and the other does not. + case( + "call_style_argument_writes_the_receiver", + "let a = [1]; push(a, { push(a, 9); 2 }); a", + ), + case( + "call_style_argument_replaces_the_receiver", + "let a = [1]; push(a, { a = [7]; 2 }); a", + ), + // The receiver appearing again among the arguments, which is where a live + // reference into the scope would be most likely to show. It does not: the + // later argument was read and flattened before the reference was taken, so + // it is a copy of what the receiver held then. + case("call_style_receiver_is_also_an_argument", "let a = [1]; push(a, a); a"), + case("call_style_receiver_twice_over", "let a = [1]; insert(a, 0, a); a"), + // Neither of these can be handed out by reference, so both are passed by + // value and the mutation is discarded (`func/call.rs:1449-1454`). + case("call_style_constant_receiver", "const a = [1]; push(a, 2); a"), + // The closure is made in a block so the scope the two sides are compared on + // does not end up holding a pointer, which they render differently on + // purpose — see `a_closure_pointer_is_late_bound` in `tests/scope.rs`. + case( + "call_style_shared_receiver", + "let a = [1]; { let f = || a.len(); } push(a, 2); a", + ), + // A script function copies its first argument whichever way it arrives, so + // the rewrite is invisible here — which is the thing to pin. + case("call_style_script_fn", "fn bump_it(x) { x += 1; x } let n = 3; bump_it(n); n"), + // The receiver resolves last, so a missing one is reported after a missing + // argument rather than before it. + case("error_receiver_resolves_after_arguments", "nosuch(receiver, argument)"), + case("error_receiver_not_found", "let ok = 1; nosuch(missing, ok)"), + // Dispatch still fails at the call, not at the variable that reached it. + case("error_no_function_for_the_receiver", "let a = [1]; nosuch(a, 2)"), + // An error a native *returned*, which rhai positions at the call site like + // everything else dispatch produces (`func/call.rs:413`). Both argument + // shapes, because one goes through the rewrite and the other does not. + case("error_returned_by_a_native", r#"parse_int("zz")"#), + case("error_returned_by_a_native_by_reference", r#"let s = "zz"; parse_int(s)"#), + + // --- closures --------------------------------------------------------- + // Function pointers are invoked via `.call()`; `f(5)` would look for a + // function literally named `f`. + // The closure is kept inside a block in all of these. Not incidental: the + // pointer we build is late-bound where rhai's is early-bound, so rhai + // renders it `Fn*+("anon$..")` and we render it `Fn("anon$..")`. That + // difference is the price of not shipping an AST body, it is script- + // visible, and `a_closure_pointer_is_late_bound` is where it is pinned — + // so these cases test the capture rather than re-testing the rendering. + case( + "closure_capture_read", + "let n = 10; let r = 0; { let f = |x| x + n; r = f.call(5); } r", + ), + // Capture is by shared cell, so the mutation must be visible outside. + case( + "closure_capture_mutate", + "let n = 0; { let f = || n += 1; f.call(); f.call(); } n", + ), + // In a block for the same reason the closure cases are: what the pointer + // *does* matches, what it renders as does not. + case( + "fn_ptr_call", + "fn triple(x) { x * 3 } let r = 0; { let f = Fn(\"triple\"); r = f.call(4); } r", + ), + // The same through a name that is not a constant, so rhai's optimizer + // cannot fold it into a pointer carrying an environment. + case( + "fn_ptr_from_dynamic_name", + "fn triple(x) { x * 3 } let n = \"trip\" + \"le\"; let f = Fn(n); f.call(4)", + ), + case( + "fn_ptr_curried", + "fn add(a, b) { a + b } let n = \"a\" + \"dd\"; let f = Fn(n).curry(10); f.call(5)", + ), + // A pointer to a native function goes to rhai's own dispatch rather than + // to a chunk of ours. + case( + "fn_ptr_to_native", + "let n = \"ab\" + \"s\"; let f = Fn(n); f.call(-7)", + ), + // Deliberately absent: `let x = 1; x.call(2)`. That is not an error in + // rhai — a non-pointer target means the *argument* is the pointer and the + // target is `this` — and the VM reproduces the behaviour, but not the + // position. Rhai blames the argument there and the call everywhere else, + // and one instruction has one position-table entry; using the argument's + // was measured to move the divergence onto the common path instead of + // removing it. A pool of positions would fix it and would not be + // strippable. + case( + "error_fn_ptr_unknown_name", + "let n = \"no\" + \"pe\"; let f = Fn(n); f.call(1)", + ), + // `Fn` and `curry` read their first argument and blame everything they can + // then complain about on *it* rather than on the call — a name that is not + // a string, a string that is not an identifier, a first argument that is + // not a pointer (`func/call.rs:1217`, `:1220`, `:1232`). + case("error_fn_ptr_from_a_non_string", "Fn(())"), + case("error_fn_ptr_from_an_unusable_name", r#"Fn("not an identifier!")"#), + case("error_curry_of_a_non_pointer", "curry(1, 2)"), + // Capturing a variable turns its slot into a shared cell, which changes + // what every later read and write of that slot means. Writing the slot + // instead of writing *through* it severs the closure silently — the value + // is right and the aliasing is dead — so this needs a write after the + // capture to catch it. + case( + "closure_shared_write", + "let x = 1; let r = 0; { let f = || x; x = 42; r = f.call(); } r", + ), + case( + "closure_shared_op_assign", + "let x = 1; let r = 0; { let f = || x; x += 41; r = f.call(); } r", + ), + // The same cell as the root of a chain. `get_indexed_mut` refuses a shared + // value outright, so walking one takes the host down rather than returning + // an error (`eval/chaining.rs:461`). + case( + "closure_shared_chain_root", + "let a = [1, 2, 3]; { let f = || a[0]; } a[1] = 20; a[1]", + ), + // rhai answers this syntactically and registers no function for it, so a + // lowered call would fail to resolve where the walker returns a bool. + case( + "is_shared_after_capture", + "let x = 1; let r = false; { let f = || x; r = is_shared(f); } [is_shared(x), r]", + ), + case("closure_in_map", "[1, 2, 3].map(|x| x * 2)"), + case("closure_in_filter", "[1, 2, 3, 4].filter(|x| x % 2 == 0)"), + + // --- chained lvalues -------------------------------------------------- + // Each of these needs a different `Target` variant and its write-back. + case("index_assign_array", "let a = [1, 2, 3]; a[1] = 99; a"), + case("index_assign_nested", "let m = #{ xs: [1, 2, 3] }; m.xs[2] = 42; m.xs"), + case("property_assign_deep", "let m = #{ a: #{ b: #{ c: 1 } } }; m.a.b.c = 7; m.a.b.c"), + case("map_autovivify", "let m = #{}; m.fresh = 1; m"), + // The other half of it: only a write creates a key. Reading one that is + // not there gives unit and must leave the map alone — the map is returned + // so the test can see whether it grew. + case("map_read_of_absent_key_does_not_create_it", "let m = #{ a: 1 }; let r = m.b; [m, r]"), + case("map_read_absent_through_a_chain", "let m = #{ a: #{} }; let r = m.a.b; [m, r]"), + // Walking through an absent key reaches a detached unit, so the write has + // nowhere to land and rhai says so rather than creating the path. + case("error_map_write_through_an_absent_key", "let m = #{}; m.a.b = 1; m"), + // A closure holds the same cell, so a key invented by a read would be + // visible from outside the expression that invented it. + case( + "map_read_of_absent_key_is_not_visible_to_a_closure", + "let m = #{}; let r = 0; { let f = || m; r = m.zz; } [m, r]", + ), + case("op_assign_indexed", "let a = [1, 2, 3]; a[0] += 10; a"), + case("bitfield_assign", "let x = 0; x[2] = true; x"), + case("string_char_assign", r#"let s = "hello"; s[0] = 'H'; s"#), + case("string_slice_read", r#"let s = "hello world"; s[0..5]"#), + // The inclusive form is a different `TypeId` and a different pool tag, so + // one does not cover the other. A string rather than an array, because + // rhai indexes arrays with integers only and slices them with `extract`. + case("string_slice_inclusive", r#"let s = "hello world"; s[6..=9]"#), + + // --- chains rooted at something that is not a variable ------------------ + // Rhai evaluates the root into a temporary and walks that + // (`eval/chaining.rs:561-571`), so there is no scope entry behind it and + // nothing is written back. One case per root shape, because each reaches a + // different `Target`. + case("temp_root_array_method", "[3, 1, 2].len()"), + case("temp_root_array_index", "[10, 20, 30][1]"), + case("temp_root_string", r#""hello".to_upper()"#), + case("temp_root_map_property", "#{ a: 1, b: 2 }.b"), + case("temp_root_call", "fn make() { [1, 2, 3] } make().len()"), + case("temp_root_parenthesised", "let a = 1; let b = 2; (a + b).to_string()"), + case("temp_root_nested", "[[1, 2], [3, 4]][1][0]"), + // A mutating method on a temporary. The mutation has nowhere to land, and + // the point is that both sides agree it is discarded rather than one of + // them inventing a place to put it. + case("temp_root_mutating_method", "let a = [1, 2, 3]; [a.len()].push(9)"), + case("temp_root_host_mutates", "widget(4).bump()"), + case("temp_root_host_pure", "widget(4).doubled()"), + // Order, which is the part that is not obvious: rhai collects a chain's + // indices *before* it evaluates what they apply to. Both halves fail, so + // the position in the reported error is which one ran first. + // An operator with no implementation for the types it got. The corpus + // reaches `ErrorFunctionNotFound` through a named call elsewhere, which is + // a different dispatch path and positions itself differently. + case("error_operator_undefined_for_types", "let a = 1.0; a + #{ b: 1 }"), + // A chain step that fails, one per kind. Rhai blames the step rather than + // the chain, and a chain is one instruction with one position-table entry, + // so these are what make each step carry its own. + case("error_property_on_a_variable", "let x = 1; x.a"), + case("error_property_on_a_temporary", "[1, 2].a"), + case("error_method_on_a_variable", "let x = 1; x.to_upper()"), + case("error_property_deep_in_a_chain", "let m = #{ a: #{} }; m.a.b.c"), + // The op-assign form, which falls back to the plain operator when no + // `+=` is registered and used to lose the position on the way. + case("error_op_assign_undefined_for_types", "let a = 1.0; a += #{ b: 1 }; a"), + case("error_temp_root_index_runs_first", "let z = 0; [1 / z][9 / z]"), + // Two positions belong to an index step, not one: the index expression, + // which an out-of-bounds is blamed on, and the `[` in front of it, which + // indexing something unindexable is blamed on. They only come apart in a + // chain of more than one step — here `n[0]` bit-indexes an integer and + // yields a bool, and rhai names the *second* `[`. + case("error_index_into_an_unindexable_step", "let n = 0; n[0][5]"), + case("error_index_into_an_unindexable_step_deep", "let m = #{ a: 1 }; m.a[0][5]"), + + // --- what an escaping error leaves in the scope ------------------------- + // Rhai rewinds a block whether it is left normally or by a throw, and + // rewinds nothing at the top level. The comparison that matters in all of + // these is the leftover scope rather than the error. + case("throw_from_a_block_rewinds_it", "let a = 1; { let b = 2; throw 3; }"), + case("throw_from_a_for_body_drops_the_loop_var", "let a = 1; for i in 0..3 { throw i; }"), + case( + "throw_from_a_while_body_drops_its_locals", + "let a = 1; let n = 0; while n < 3 { let b = n; throw b; }", + ), + // A catch block is a block too, and its variable is the one rhai pushes + // rather than the script. + case("throw_from_a_catch_drops_the_catch_var", "let a = 1; try { throw 2; } catch (e) { throw e; }"), + case( + "throw_from_a_nested_block_drops_every_level", + "let a = 1; { let b = 2; { let c = 3; for i in 0..2 { throw i; } } }", + ), + // The frame boundary: a function's own locals go with its scope, and the + // caller's top-level ones stay. + case( + "throw_from_a_function_leaves_the_caller_top_level_alone", + "fn boom() { let inner = 9; throw inner; } let a = 1; { let b = 2; boom(); }", + ), + // Nothing to rewind, which is the case a floor set too low would break. + case("throw_at_the_top_level_keeps_what_ran", "let a = 1; let b = 2; throw 3;"), + case("error_temp_root_out_of_bounds", "[1, 2, 3][99]"), + + // --- errors ----------------------------------------------------------- + // Compared by variant and position, so a VM that reports the right failure + // at the wrong place still fails the test. + case("error_unknown_variable", "let a = 1; a + nonexistent"), + // Positions on call failures are set by different code paths depending on + // whether the callee was found, so both need pinning. + case("error_unknown_function", "let a = 1; no_such_function(a)"), + case("error_wrong_arity", "fn f(a, b) { a } f(1)"), + case("error_array_bounds", "let a = [1, 2]; a[10]"), + // Rhai maps the *expected* type through its registered names and leaves the + // *actual* one raw, so a range guard reports `core::ops::range::Range` + // rather than the `range` the same engine prints everywhere else. Mapping + // both is the obvious mistake, and only a type with a registered name shows + // it up. + case("error_condition_is_a_range", "if 0..1 { 1 } else { 2 }"), + case("error_condition_is_a_host_type", "let w = widget(1); while w { 1 }"), + case("error_type_mismatch",r#"let a = 1; a + "string" + [1]"#), + case("error_divide_by_zero", "let a = 1; a / 0"), + case("throw_value", "throw 42"), + case("throw_in_fn", "fn f() { throw \"boom\"; } f()"), + + // --- try / catch ------------------------------------------------------ + case("try_catch_value", "try { throw 7; } catch (e) { e * 2 }"), + case("try_catch_native_error", "try { let a = [1]; a[9] } catch (e) { e.message != () }"), + case("try_catch_rethrow", "try { try { throw 1; } catch { throw; } } catch (e) { e }"), + // `return` unwinds as an error but must pass straight through a catch. + case( + "try_catch_does_not_swallow_return", + "fn f() { try { return 1; } catch { return 2; } } f()", + ), + case("try_catch_no_error", "try { 5 } catch { 6 }"), + // The catch block's value is discarded — the statement is unit on the + // caught path and the try block's value otherwise (`eval/stmt.rs:863`). + case("try_catch_discards_its_value", "try { throw 1; } catch { 99 }"), + // A jump out of a `try` skips the `PopHandler` the straight-line path + // would have run. Left armed, the next error is caught into a block that + // has already been left — so the second failure here must not be caught. + case( + "break_out_of_try_disarms_it", + "let s = 0; while true { try { throw 1; } catch { break; } } try { throw 2; } catch (e) { s = e; } s", + ), + case( + "break_out_of_for_inside_try", + "let s = 0; for i in 0..5 { try { if i == 2 { break; } s += i; } catch { s = -1; } } s", + ), + case( + "continue_out_of_try_inside_for", + "let s = 0; for i in 0..5 { try { if i % 2 == 0 { continue; } s += i; } catch { s = -1; } } s", + ), + // An error out of a called function arrives wrapped in + // `ErrorInFunctionCall`, which is catchable, and `unwrap_inner` is what + // still binds the bare thrown value. + case( + "try_around_a_compiled_call", + "fn boom() { throw 7; } try { boom(); } catch (e) { e }", + ), + // `return` is a pseudo error and must pass straight through a handler. + case( + "try_does_not_catch_return", + "fn f() { try { return 1; } catch { 2 } } f()", + ), + + // --- host types ------------------------------------------------------- + // + // The one part of the chain walker that approximates rather than + // reproduces. A getter hands back a value, so anything below it mutates a + // temporary that only the setter can put back — and rhai decides whether + // to call the setter from `func.is_method()`, which is not visible from + // outside the crate. + case("host_get", "let w = widget(4); w.level"), + case("host_set", "let w = widget(4); w.level = 9; w.level"), + case("host_op_assign", "let w = widget(4); w.level += 5; w.level"), + case("host_index_get", "let w = widget(1); w[1]"), + case("host_index_set", "let w = widget(1); w[1] = 99; w[1]"), + case("host_method_mutates", "let w = widget(4); w.bump(); w.level"), + case("host_method_pure", "let w = widget(4); w.doubled()"), + // Two levels, so the middle one is a temporary. + case("host_temp_set", "let h = holder(3); h.inner.level = 8; h.inner.level"), + case( + "host_temp_index_set", + "let h = holder(3); h.inner[0] = 7; h.inner[0]", + ), + // A mutating call on a temporary: rhai writes it back, so the change + // survives. + case("host_temp_mutates", "let h = holder(3); h.inner.bump(); h.inner.level"), + // A read-only call on a temporary, which is where rhai's own flag decides + // whether a setter runs at all. + case("host_temp_pure", "let h = holder(3); h.inner.doubled()"), + case("error_host_index_bounds", "let w = widget(1); w[99]"), +]; diff --git a/tests/grain/differential.rs b/tests/grain/differential.rs new file mode 100644 index 000000000..8690802ba --- /dev/null +++ b/tests/grain/differential.rs @@ -0,0 +1,363 @@ +//! The VM must mean exactly what rhai means. +//! +//! Every corpus script is evaluated twice against the same `Engine` — once +//! through `eval_ast_with_scope`, once through the VM — and the two runs must +//! agree on the result, on the error (variant *and* position), and on the scope +//! they leave behind. +//! +//! The scope check is not incidental. Rhai evaluates a program's top-level +//! statements without rewinding, so `let` at the top level outlives the run and +//! is observable by the caller. A VM that manages its own frames could return +//! the right value and still get that wrong. + +mod corpus; + +use rhai::{Dynamic, Engine, Scope}; +use rhaigrain::{Compiler, Vm}; + +/// What a run produced, in a form two runs can be compared on. +/// +/// `Dynamic` and `EvalAltResult` have no `PartialEq`, so this compares their +/// `Debug` rendering. That is stricter than value equality, not looser: it +/// distinguishes `1` from `1.0`, and it includes error positions. +#[derive(Debug, PartialEq, Eq)] +struct Outcome { + result: Result, + scope: Vec<(String, String)>, +} + +fn snapshot_scope(scope: &Scope) -> Vec<(String, String)> { + scope + .iter_raw() + .map(|(name, _, value)| (name.to_string(), format!("{value:?}"))) + .collect() +} + +fn run_stock(engine: &Engine, source: &str) -> Outcome { + let mut scope = Scope::new(); + let result = engine + .compile(source) + .map_err(|err| format!("{err:?}")) + .and_then(|ast| { + engine + .eval_ast_with_scope::(&mut scope, &ast) + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")) + }); + + Outcome { + result, + scope: snapshot_scope(&scope), + } +} + +fn run_vm(engine: &Engine, source: &str) -> Outcome { + let mut scope = Scope::new(); + let result = engine + .compile(source) + .map_err(|err| format!("{err:?}")) + .and_then(|ast| { + let program = Compiler::new().compile(&ast); + // A program that can hand a pointer to a native has to be run the + // way such a program is meant to be run, or the comparison is + // against a configuration nobody would ship. + if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(engine).run_with_callbacks(&program, &mut scope) + } else { + Vm::new(engine).run(&program, &mut scope) + } + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")) + }); + + Outcome { + result, + scope: snapshot_scope(&scope), + } +} + +#[test] +fn vm_agrees_with_rhai() { + let engine = corpus::engine(); + + let mut failures = Vec::new(); + + for case in corpus::CASES { + let stock = run_stock(&engine, case.source); + let vm = run_vm(&engine, case.source); + + if stock != vm { + failures.push(format!( + "\n=== {} ===\n source: {}\n rhai: {:?}\n vm: {:?}\n \ + rhai scope: {:?}\n vm scope: {:?}", + case.name, case.source, stock.result, vm.result, stock.scope, vm.scope, + )); + } + } + + assert!( + failures.is_empty(), + "{} of {} corpus scripts diverged:{}", + failures.len(), + corpus::CASES.len(), + failures.join(""), + ); +} + +/// The corpus is only worth anything if the comparison can actually fail. +/// +/// Guards against the harness silently degrading into a tautology — comparing +/// two identical code paths, or stringifying everything into the same value. +#[test] +fn harness_detects_a_real_difference() { + let engine = corpus::engine(); + + assert_ne!( + run_stock(&engine, "1 + 1"), + run_stock(&engine, "1 + 2"), + "differing results must compare unequal", + ); + assert_ne!( + run_stock(&engine, "1"), + run_stock(&engine, "1.0"), + "int and float must not compare equal", + ); + assert_ne!( + run_stock(&engine, "let a = 1; a"), + run_stock(&engine, "1"), + "differing leftover scope must compare unequal", + ); + assert_ne!( + run_stock(&engine, "let a = [1]; a[9]"), + run_stock(&engine, "let a = [1]; a[9]"), + "the same error at a different position must compare unequal", + ); +} + +/// A script that does not parse compares equal on both sides for the wrong +/// reason: two identical parse errors. Such a case tests nothing, and would sit +/// in the corpus looking like coverage. +#[test] +fn every_corpus_script_parses() { + let engine = corpus::engine(); + + let broken: Vec<_> = corpus::CASES + .iter() + .filter_map(|case| { + engine + .compile(case.source) + .err() + .map(|err| format!("\n {}: {err}", case.name)) + }) + .collect(); + + assert!( + broken.is_empty(), + "{} corpus scripts do not parse, so they assert nothing:{}", + broken.len(), + broken.join(""), + ); +} + +/// Whether a corpus case exercises anything on this build. +/// +/// Distinct from [`MAY_FRAGMENT`], which is a tolerance. A restriction feature +/// removes the syntax outright — rhai will not parse a capturing closure under +/// `no_closure` — so the case tests nothing here, and both sides agreeing on +/// the parse failure would be an empty agreement rather than a passing one. +fn applies_to_this_build(name: &str) -> bool { + #[cfg(feature = "no_closure")] + if name.starts_with("closure_") || name.starts_with("is_shared") { + return false; + } + #[cfg(feature = "no_module")] + if name.starts_with("import_") || name.starts_with("export_") { + return false; + } + let _ = name; + true +} + +/// A case that errors unintentionally is nearly as weak as one that does not +/// parse: both sides agree on the failure, and the machinery the case was +/// written to exercise never runs. Cases that mean to fail say so in the name. +#[test] +fn only_error_cases_error() { + let engine = corpus::engine(); + + let surprises: Vec<_> = corpus::CASES + .iter() + .filter(|case| !case.name.starts_with("error_") && !case.name.starts_with("throw_")) + .filter(|case| applies_to_this_build(case.name)) + .filter_map(|case| match run_stock(&engine, case.source).result { + Err(err) => Some(format!("\n {}: {err}", case.name)), + Ok(_) => None, + }) + .collect(); + + assert!( + surprises.is_empty(), + "{} cases fail without meaning to, so they exercise nothing:{}", + surprises.len(), + surprises.join(""), + ); +} + +/// Corpus scripts allowed to leave a fragment behind. +/// +/// Empty, and that is the claim: every script in the corpus lowers with nothing +/// left over. A case that fragments therefore fails on arrival rather than +/// quietly joining a majority, which is the point of stating it this way round. +/// +/// What legitimately belongs here: `eval`, `import`/`export`, custom syntax, +/// and `?.`. All four are the escape hatch working as intended rather than a +/// gap, and none is in the corpus. +const MAY_FRAGMENT: &[&str] = &[]; + +/// Every chunk the compiler emits must pass its own verifier. +/// +/// The check that matters is depth agreement at merge points: one branch of a +/// conditional leaving a value where the other does not is invisible until a +/// program happens to take the unlucky path, and the differential corpus only +/// covers the paths it happens to exercise. +#[test] +fn every_compiled_chunk_verifies() { + let engine = corpus::engine(); + + let broken: Vec<_> = corpus::CASES + .iter() + .filter_map(|case| { + let ast = engine.compile(case.source).ok()?; + Compiler::new() + .compile(&ast) + .verify() + .err() + .map(|err| format!("\n {}: {err:?}", case.name)) + }) + .collect(); + + assert!( + broken.is_empty(), + "{} chunks failed verification:{}", + broken.len(), + broken.join(""), + ); +} + +/// A chunk must declare the stack it uses, not the stack it might use. +/// +/// The lowering's own estimate is one slot per instruction, which is safe and +/// wildly loose — and it is what the VM reserves from, and what the artifact +/// records. On a device with ~12KB to spend, reserving 25 `Dynamic` slots for a +/// chunk that stacks three is the difference worth closing. +#[test] +fn every_compiled_chunk_declares_the_stack_it_uses() { + let engine = corpus::engine(); + + let mut loose = Vec::new(); + let mut total_declared = 0usize; + let mut total_ops = 0usize; + + for case in corpus::CASES { + let Ok(ast) = engine.compile(case.source) else { + continue; + }; + let program = Compiler::new().compile(&ast); + let Ok(high_water) = program.verify() else { + continue; + }; + + let declared: Vec = std::iter::once(program.main().max_stack()) + .chain(program.functions().iter().map(|f| f.chunk.max_stack())) + .collect(); + + total_declared += high_water.iter().map(|n| *n as usize).sum::(); + total_ops += program.code().len(); + + if declared != high_water { + loose.push(format!( + "\n {}: declares {declared:?}, uses {high_water:?}", + case.name, + )); + } + } + + println!("\n{total_declared} stack slots declared across {total_ops} bytes of code"); + + assert!( + loose.is_empty(), + "{} chunks declare a stack they do not use:{}", + loose.len(), + loose.join(""), + ); +} + +/// Residuals are the work left to do, so the count is the progress metric. +/// +/// Prints the whole census so a milestone's effect is visible, and pins the +/// cases that should already be at zero. +#[test] +fn residual_census() { + let engine = corpus::engine(); + + let mut total_nodes = 0usize; + let mut at_zero = Vec::new(); + let mut remaining = Vec::new(); + let mut regressions = Vec::new(); + + // Cases the build removed are counted on neither side, or the completeness + // check below would read their absence as a corpus that stopped compiling. + let applicable = corpus::CASES + .iter() + .filter(|case| applies_to_this_build(case.name)) + .count(); + + for case in corpus::CASES.iter().filter(|c| applies_to_this_build(c.name)) { + let Ok(ast) = engine.compile(case.source) else { + continue; + }; + let program = Compiler::new().compile(&ast); + let count = program.residual_count(); + let nodes = program.residual_nodes(); + total_nodes += nodes; + + if count == 0 { + at_zero.push(case.name); + } else { + remaining.push((case.name, nodes)); + if !MAY_FRAGMENT.contains(&case.name) && applies_to_this_build(case.name) { + regressions.push(format!("\n {} leaves {count}", case.name)); + } + } + } + + println!( + "\n{} of {} scripts fully lowered, {total_nodes} AST nodes still in fragments", + at_zero.len(), + corpus::CASES.len(), + ); + println!("\nfully lowered: {}", at_zero.join(", ")); + println!("\nremaining:"); + remaining.sort_by_key(|(_, count)| std::cmp::Reverse(*count)); + for (name, count) in &remaining { + println!(" {count:>3} {name}"); + } + + assert!( + regressions.is_empty(), + "{} scripts fragment that are not on `MAY_FRAGMENT`. Either the \ + construct regressed, or the case needs one of the four things the \ + escape hatch is for — say which, in the list:{}", + regressions.len(), + regressions.join(""), + ); + + // The other direction, which the check above cannot see: a corpus that + // stopped compiling at all would have nothing to fragment and would pass. + assert_eq!( + at_zero.len(), + applicable - MAY_FRAGMENT.len(), + "some scripts did not compile, so they were counted as neither", + ); +} diff --git a/tests/grain/fixtures/follow.rhai b/tests/grain/fixtures/follow.rhai new file mode 100644 index 000000000..1e9d45480 --- /dev/null +++ b/tests/grain/fixtures/follow.rhai @@ -0,0 +1,296 @@ +fn pickn(ord, start, k) { + let out = [false, false, false]; + let i = 0; + while i < k && i < 3 { + out[ord[(start + i) % 3]] = true; + i += 1; + } + out +} + +fn order3(a, b, c) { + let ord = [0, 1, 2]; + let lev = [a, b, c]; + if lev[ord[1]] > lev[ord[0]] { let t = ord[0]; ord[0] = ord[1]; ord[1] = t; } + if lev[ord[2]] > lev[ord[1]] { let t = ord[1]; ord[1] = ord[2]; ord[2] = t; } + if lev[ord[1]] > lev[ord[0]] { let t = ord[0]; ord[0] = ord[1]; ord[1] = t; } + ord +} + +let STEP = 20; +let PEAK_DECAY = 0.999; +let PEAK_FLOOR = 20.0; +let FLUX_SMOOTH = 0.30; +let THR_SMOOTH = 0.02; +let THR_MULT = 1.8; +let THR_BIAS = 0.03; +let REFRACT = 110; +let BEAT_MIN = 380; +let BEAT_MAX = 680; +let BEAT_GAP = 6000; +let AGREE_PCT = 25; +let LOOK_BEATS = 8; +let DEAD_MS = 8000; +let PATTERN_MAX = 700; +let DIMMER_CH = 4; +let PAN_CH = 6; +let HOT_CH = 11; +let PHRASE_MIN = 4000; +let VOTE_SEQ = 3; +let COLOUR_DEAD = 0.30; +let WIDE_AT = 0.18; +let FULL_AT = 0.45; + +let DWELL = lamp_dwell_ms(); +if DWELL < 1 { DWELL = 1; } + +let pk_r = PEAK_FLOOR; +let pk_g = PEAK_FLOOR; +let pk_b = PEAK_FLOOR; +let pk_e = PEAK_FLOOR; +let e_prev = 0.0; +let flux = 0.0; +let thr = 0.05; +let lvl = 0.0; + +let last_onset = -99999; +let period = 500; +let anchor = 0; +let disagree = 0; +let last_seq = 0; + +let ord = [0, 1, 2]; +let pan_s = 0.0; +let pan_prev = 0.0; +let pan_dir = 0; +let pan_hi = 0.0; +let pan_lo = 0.0; +let pan_ref = 0.0; +let phrase_at = -99999; +let look_at = -99999; +let hot = false; +let gated = false; + +let beat_count = 0; +let last_slot = -1; +let scatter_start = 0; +let scatter_w = 1; + +let look = 0; +let look_until = 0; +let accent_until = -99999; + +let cur_r = false; +let cur_y = false; +let cur_g = false; +let last_change = -99999; + +let nudge = false; +let nudge_s = [false, false, false]; + +let t0 = millis(); +let next = t0; +let last_pkt = t0; +anchor = t0; + +loop { + let p = dmx_recv(STEP); + let now = millis(); + + if p.ok { + last_pkt = now; + + let dseq = p.seq - last_seq; + if dseq < 1 { dseq = 1; } + if dseq > 32 { dseq = 32; } + last_seq = p.seq; + + let r = p.ch[0].to_float(); + let g = p.ch[1].to_float(); + let b = p.ch[2].to_float(); + + pk_r *= PEAK_DECAY; + pk_g *= PEAK_DECAY; + pk_b *= PEAK_DECAY; + if pk_r < PEAK_FLOOR { pk_r = PEAK_FLOOR; } + if pk_g < PEAK_FLOOR { pk_g = PEAK_FLOOR; } + if pk_b < PEAK_FLOOR { pk_b = PEAK_FLOOR; } + if r > pk_r { pk_r = r; } + if g > pk_g { pk_g = g; } + if b > pk_b { pk_b = b; } + + let lr = r / pk_r; + let lb = b / pk_b; + let lg = g / pk_g; + let cmax = lr; + if lb > cmax { cmax = lb; } + if lg > cmax { cmax = lg; } + if cmax < COLOUR_DEAD { + let s0 = beat_count % 3; + ord = [s0, (s0 + 1) % 3, (s0 + 2) % 3]; + } else { + ord = order3(lr, lb, lg); + } + + let raw = r; + if g > raw { raw = g; } + if b > raw { raw = b; } + if p.ch.len() > DIMMER_CH { raw = p.ch[DIMMER_CH].to_float(); } + pk_e *= PEAK_DECAY; + if pk_e < PEAK_FLOOR { pk_e = PEAK_FLOOR; } + if raw > pk_e { pk_e = raw; } + let e = raw / pk_e; + + if p.ch.len() > PAN_CH { + pan_s += (p.ch[PAN_CH].to_float() - pan_s) * 0.08; + if pan_s > pan_hi { pan_hi = pan_s; } else { pan_hi += (pan_s - pan_hi) * 0.005; } + if pan_s < pan_lo { pan_lo = pan_s; } else { pan_lo += (pan_s - pan_lo) * 0.005; } + let dir = pan_dir; + if pan_s > pan_prev + 0.4 { dir = 1; } + else if pan_s < pan_prev - 0.4 { dir = -1; } + if dir != pan_dir && pan_dir != 0 && now - phrase_at > PHRASE_MIN { + let span = pan_hi - pan_lo; + let trav = pan_s - pan_ref; + if trav < 0.0 { trav = -trav; } + if span > 8.0 && trav > span * 0.35 { + phrase_at = now; + pan_ref = pan_s; + } + } + pan_dir = dir; + pan_prev = pan_s; + } + + if p.ch.len() > HOT_CH + 2 { + gated = true; + let h = p.ch[HOT_CH]; + if p.ch[HOT_CH + 1] > h { h = p.ch[HOT_CH + 1]; } + if p.ch[HOT_CH + 2] > h { h = p.ch[HOT_CH + 2]; } + hot = h > 24; + } + + let d = (e - e_prev) / dseq.to_float(); + if d < 0.0 { d = 0.0; } + e_prev = e; + flux += (d - flux) * FLUX_SMOOTH; + lvl += (e - lvl) * 0.06; + + if flux > thr * THR_MULT + THR_BIAS && now - last_onset >= REFRACT + && dseq <= VOTE_SEQ { + let iv = now - last_onset; + last_onset = now; + accent_until = now + DWELL; + + if iv <= BEAT_GAP { + while iv < BEAT_MIN { iv *= 2; } + while iv > BEAT_MAX { iv /= 2; } + if iv >= BEAT_MIN { + let err = iv - period; + if err < 0 { err = -err; } + if err * 100 <= period * AGREE_PCT { + period = (period * 4 + iv) / 5; + let ph = (now - anchor) % period; + if ph * 2 > period { ph -= period; } + let mag = ph; + if mag < 0 { mag = -mag; } + if mag * 4 > period { anchor = now; } else { anchor += ph / 4; } + disagree = 0; + } else { + disagree += 1; + if disagree >= 3 { + period = iv; + anchor = now; + disagree = 0; + } + } + } + } + } + thr += (flux - thr) * THR_SMOOTH; + } + + let pd = period; + while pd > PATTERN_MAX { pd /= 2; } + while pd < DWELL * 4 { pd *= 2; } + + let since = now - anchor; + if since < 0 { since = 0; } + let frac = (since % pd) * 1000 / pd; + let quarter = frac / 250; + let half = frac / 500; + + let live = now - last_pkt < DEAD_MS; + let wide_at = WIDE_AT; + let full_at = FULL_AT; + if gated && hot { + wide_at *= 0.6; + full_at *= 0.7; + } + let w = 1; + if lvl > wide_at { w = 2; } + if lvl > full_at { w = 3; } + if live && w < 2 { w = 2; } + let rest = w - 1; + if rest < 1 { rest = 1; } + + if quarter != last_slot { + if quarter == 0 { beat_count += 1; } + nudge = false; + if quarter % 2 == 0 { + scatter_start = rand_int(0, 2); + scatter_w = rand_int(1, w); + } + last_slot = quarter; + } + + if beat_count >= look_until || phrase_at > look_at { + look_at = phrase_at; + look_until = beat_count + LOOK_BEATS; + look = rand_int(0, 5); + } + + let s = [false, false, false]; + + if !live { + s = pickn(ord, beat_count, 2); + } else if now < accent_until { + s = pickn(ord, 0, 3); + } else if look == 0 { + if quarter < 2 { s = pickn(ord, 0, w); } else { s = pickn(ord, 0, rest); } + } else if look == 1 { + s = pickn(ord, beat_count * 2 + half, 1); + } else if look == 2 { + if quarter == 0 || quarter == 2 { s = pickn(ord, 0, w); } else { s = pickn(ord, half, rest); } + } else if look == 3 { + if quarter >= 2 { s = pickn(ord, half, w); } else { s = pickn(ord, beat_count, rest); } + } else if look == 4 { + let k = quarter + 1; + if k > 3 { k = 1; } + if k > w + 1 { k = w + 1; } + s = pickn(ord, 0, k); + } else { + s = pickn(ord, scatter_start, scatter_w); + } + + if !nudge && now - last_change >= pd / 2 { + nudge = true; + if cur_r || cur_y || cur_g { + nudge_s = [false, false, false]; + } else { + let k = w; + if k < 1 { k = 1; } + nudge_s = pickn(ord, beat_count, k); + } + } + if nudge { s = nudge_s; } + + if (s[0] != cur_r || s[1] != cur_y || s[2] != cur_g) && now - last_change >= DWELL { + set_lights(s[0], s[1], s[2]); + cur_r = s[0]; cur_y = s[1]; cur_g = s[2]; + last_change = now; + } + + next += STEP; + if next < now { next = now; } + sleep_until(next); +} diff --git a/tests/grain/fixtures/golden.rgrn b/tests/grain/fixtures/golden.rgrn new file mode 100644 index 0000000000000000000000000000000000000000..d71f299a38cd87ae0309d3972a2b5e7035f396de GIT binary patch literal 1759 zcmYLJOK%%h6#mY=Gk2c0XKZJZrgq|(Bz>fXI;JFu)-)89qKFcb@Jbip#FNCrv4cIP ztppM~cCg~LsDK3vmaGsPgjj$Of1rP$690h>=Z>9re0A@49`~H@oH^sI^{v|qwhjH9 zQ+^nooteK>yZXY-jrR|`y_5dn{>PtuN}ti^^d)^oU(>hr9sNi@(a-cN{YHP#U-S?E zws*E(efgFB-msetI?cVf({Cp2-a)5%IOxRPr1xICo9y?FPlid{N}8R{{gdI~=x)0e z50b&~wEWY~{e&2FzdzgJ&wuC!`b;%27b$h-_Bq);k3>i6z;<)381MIe=( zu}`T(jTpl|Z3GP6AQ(q+vxPJ+%(QQ?2BQjKpLPdE(rkBlkMAWgQ?gK_211Qwl^T^% z=^a$mDA1TUZm2YsN|UI^LDe{d8dFn`$ub(hEp^I8r9532oQJ7+LK$?88g!ACNGx5? z(^8>GL?CGbr3#Uhj1}X{tma2Lgo;gK)V}EaYlxicBI;V6t=f?i!nc1yyj4N~1?T{1iui>ob>fyiWCq;S>I9&85A zczBLhSoXAR!SS{4We=A$iDxoc^}xh02q(+3X=4eN1$GMU@r9hepl=2mWg&>3g zvj6dipcS_M%gC|7r89OAq}cykA3m4Nv4m690p@pDngD0wm_ek8X0EU*az>~0HEkn% zgLiUJ68;(^j_V?FVY%QaRezs#rZ*P7&_h*|^x#bE9OR=coU@quh21JX@}>{UjI(9G z*YRdUAO7ze`9J90(=I?zf)I$fS^h^XvXe-?IA#`<)3oc%E5L#&ut*^mBOiO7Qr zm^|Tyafmw4brDepbyiLw%A(E#8X?M|Zh|IZD%2Hd3Z{#?1v&%MLY=!Z4bw#30iD$q z)?H8uR7Mn_&Qmu7nne_%?&H}E3Q-;rrz}876igni1zies`HPtOg5Oa^a`_Urz{2RA z3|SfSy!qcS@VJb^zw}zI6w1FAicWPxRvq(vL7pe?f-)~qSc_!s zoLMVK9=2M+U7ixl+49Piu@cG^N3A&aY9v=P{_|G(B{$OuR3k7OCD|zXjiUXcAab7-pV*{=bU$P&bwJ>H;}u5b0?6zZj(`)k+sXRwvZ1(GYIWq(g<_^19G7K AO8@`> literal 0 HcmV?d00001 diff --git a/tests/grain/fixtures/golden.rhai b/tests/grain/fixtures/golden.rhai new file mode 100644 index 000000000..6ad31fbb4 --- /dev/null +++ b/tests/grain/fixtures/golden.rhai @@ -0,0 +1,92 @@ +// The script behind `tests/fixtures/golden.rgrn`. See +// `an_artifact_written_by_an_older_build_still_runs` in `tests/format.rs`. +// +// Deliberately wide rather than deliberately short: every encoder branch this +// reaches is a branch the golden artifact pins against silent drift. Editing it +// means regenerating the artifact, which is the point — the pair only proves +// anything while the artifact is older than the reader. +// +// No `switch`. A switch table carries hashes from rhai's per-process hasher +// seed, so an artifact holding one is refused by any process that did not set +// the same seed (`format/write.rs:237`). That is right for a real artifact and +// useless for a fixture, which has to load anywhere. + +fn tally(items, bias) { + let total = bias; + for item in items { + total += item; + } + total +} + +const STRIDE = 2; +let counts = [1, 2, 3]; +let label = "measure"; +let ratio = 2.5; +let flags = #{ on: true, off: false, mark: 'x' }; +// Not all-constant, so the optimizer leaves it to be built at run time. +let computed = [STRIDE, counts.len(), ratio.to_int()]; + +// A chain rooted at a slot, one on a temporary, and one on the name the caller +// supplied — the three `Root` variants. +counts.push(tally(counts, 10)); +let widest = [9, 8].len(); +supplied.push(counts.len()); + +// The function-call spelling of the same method, which takes its first +// argument by reference — once over a slot and once over the caller's name, +// where the receiver is resolved after the other argument and moved back under +// it. +push(counts, ratio.to_int()); +push(supplied, STRIDE); + +// A closure, which shares what it captures and curries it onto a pointer. Made +// and spent inside a block, so the scope this is compared on does not end up +// holding a pointer — the two sides render one differently on purpose, which +// `a_closure_pointer_is_late_bound` in `tests/scope.rs` pins. +let captured = 0; +{ + let capture = || supplied.len() + counts.len(); + captured = capture.call(); +} + +let doubled = 0; +let i = 0; +while i < counts.len() { + doubled += counts[i]; + i += 1; +} + +let caught = (); +try { + if doubled > 0 || flags.off { + throw "thrown"; + } + caught = counts[99]; +} catch (err) { + caught = err; +} + +let squares = []; +for n in 0..8 { + if n > 3 { + break; + } + squares.push(n * STRIDE); +} + +#{ + total: doubled, + text: `${label} of ${counts.len()}`, + ratio: ratio * 2.0, + flag: flags.on, + mark: flags.mark, + supplied: supplied.len(), + widest: widest, + caught: caught, + captured: captured, + computed: computed, + squares: squares, + head: label[0..2], + tail: label[1..=3], +} diff --git a/tests/grain/format.rs b/tests/grain/format.rs new file mode 100644 index 000000000..47015ec51 --- /dev/null +++ b/tests/grain/format.rs @@ -0,0 +1,768 @@ +//! An artifact must mean what the program it came from meant, and nothing a +//! wire can hand it may take the process down. +//! +//! Two separate claims, and they need separate tests. The first is a round +//! trip: compile, write, read back, run, and get what running the original +//! got — including the scope left behind and the exact error position. The +//! second is that `read` is total over arbitrary bytes: every truncation and +//! every single-byte corruption of a valid artifact either loads or fails, and +//! never panics. + +mod corpus; + +use rhai::{Dynamic, Engine, Scope}; +use rhaigrain::format::{ReadError, WriteError}; +use rhaigrain::{Compiler, Program, Vm}; + +/// What a run produced, in a form two runs can be compared on. +#[derive(Debug, PartialEq, Eq)] +struct Outcome { + result: Result, + scope: Vec<(String, String)>, +} + +/// A finished run, reduced to what two of them can be compared on. +fn snapshot(scope: &Scope, result: Result>) -> Outcome { + Outcome { + result: result + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")), + scope: scope + .iter_raw() + .map(|(name, _, value)| (name.to_string(), format!("{value:?}"))) + .collect(), + } +} + +/// Taken by value, because a program that hands pointers to natives has to be +/// owned to be run at all — and whether this one does is read back off the +/// bytes, which is the property that makes an artifact self-describing. +fn run(engine: &Engine, program: Program) -> Outcome { + let mut scope = Scope::new(); + let result = if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(engine).run_with_callbacks(&program, &mut scope) + } else { + Vm::new(engine).run(&program, &mut scope) + }; + + snapshot(&scope, result) +} + +fn run_stock(engine: &Engine, source: &str) -> Outcome { + let mut scope = Scope::new(); + let ast = engine.compile(source).expect("corpus scripts parse"); + let result = engine.eval_ast_with_scope::(&mut scope, &ast); + + snapshot(&scope, result) +} + +/// Every corpus script that can be written, with its bytes. +fn writable(engine: &Engine) -> Vec<(&'static str, &'static str, Vec)> { + corpus::CASES + .iter() + .filter_map(|case| { + let ast = engine.compile(case.source).ok()?; + let bytes = Compiler::new().compile(&ast).write().ok()?; + Some((case.name, case.source, bytes)) + }) + .collect() +} + +/// The claim the format exists to support: bytes in one process mean the same +/// program in another. +#[test] +fn an_artifact_runs_as_the_program_it_came_from() { + let engine = corpus::engine(); + let mut failures = Vec::new(); + + for (name, source, bytes) in writable(&engine) { + let reloaded = match Program::read(&bytes) { + Ok(program) => program, + Err(err) => { + failures.push(format!("\n {name}: wrote but could not read back: {err}")); + continue; + } + }; + + let expected = run_stock(&engine, source); + let actual = run(&engine, reloaded); + + if expected != actual { + failures.push(format!( + "\n {name}: {source}\n rhai: {expected:?}\n artifact: {actual:?}" + )); + } + } + + assert!( + failures.is_empty(), + "{} artifacts do not mean what they came from:{}", + failures.len(), + failures.join(""), + ); +} + +/// A round trip over an empty set passes trivially, so pin the size of the set +/// and pin that the things it should contain are in it. +#[test] +fn the_round_trip_covers_something_worth_covering() { + let engine = corpus::engine(); + let written = writable(&engine); + let names: Vec<_> = written.iter().map(|(name, ..)| *name).collect(); + + assert!( + written.len() >= 20, + "only {} corpus scripts are writable, which is too few to prove anything: {names:?}", + written.len(), + ); + + // One per construct the encoder has a branch for, so a branch that stops + // working names itself. + for required in [ + "int_arithmetic", // Call with an operator token + "float_arithmetic", // a float constant, whose width the ABI pins + "shadowing_nested", // DeclareLocal and UnwindTo + "while_loop", // jumps, Tick, AssignLocal with an op + "loop_break_value", // backpatched jumps + "error_divide_by_zero", // a position that has to survive + "switch_range", // a switch table, and the hasher probe with it + "switch_guard", // and one whose arms are a chain rather than a target + "string_slice_read", // a range constant, which is a host type in `Dynamic` + "string_slice_inclusive", // and the other range tag + "index_assign_array", // a chain rooted at a slot, and its name + "temp_root_array_method", // and one rooted on the operand stack instead + ] { + assert!( + names.contains(&required), + "`{required}` no longer writes, so the encoder branch it covers is untested", + ); + } +} + +/// Where the golden pair lives. The source is checked in beside the artifact so +/// a regeneration is a visible two-file change. +const GOLDEN_SOURCE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/golden.rhai"); +const GOLDEN_ARTIFACT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/golden.rgrn"); + +/// The caller state `golden.rhai` expects. Part of the fixture, so it lives +/// with it rather than being invented at each use. +fn golden_scope() -> Scope<'static> { + let mut scope = Scope::new(); + scope.push("supplied", vec![Dynamic::from(7_i64)]); + scope +} + +/// The one claim every other test in this file is blind to: that an artifact +/// written *earlier* still means the same thing. +/// +/// Named for `golden` so the regeneration command below selects it and nothing +/// else. +/// +/// Every other artifact here is produced by the current writer in the same +/// process, so a writer and reader that drift together agree with each other +/// perfectly and nothing notices. The device is the case that matters — bytes +/// built by one version of this crate and run by another — and a checked-in +/// artifact is the only way to have one side of that be genuinely old. +/// +/// Failing this is not automatically a bug. It means the encoding moved, and +/// the question it asks is whether that was deliberate. If it was, regenerate: +/// +/// ```text +/// REGENERATE_GOLDEN=1 cargo test --test format golden +/// ``` +/// +/// and bump `VERSION` if an older reader would *misread* the new bytes rather +/// than reject them — the rule is at `src/format/mod.rs:56`. +#[test] +fn a_golden_artifact_written_by_an_older_build_still_runs() { + let engine = corpus::engine(); + let source = std::fs::read_to_string(GOLDEN_SOURCE).expect("the golden source is checked in"); + let ast = engine.compile(&source).expect("the golden source must parse"); + let program = Compiler::new().compile(&ast); + assert_eq!( + program.residual_count(), + 0, + "the golden source must lower whole, or the artifact covers less than it claims: {:?}", + program.first_unsupported(), + ); + + if std::env::var_os("REGENERATE_GOLDEN").is_some() { + let bytes = program.write().expect("the golden source must be writable"); + std::fs::write(GOLDEN_ARTIFACT, &bytes).expect("must write the artifact"); + println!("wrote {} bytes to {GOLDEN_ARTIFACT}", bytes.len()); + return; + } + + let bytes = std::fs::read(GOLDEN_ARTIFACT).expect("the golden artifact is checked in"); + let loaded = match Program::read(&bytes) { + Ok(loaded) => loaded, + // The header records the ABI the fixture was written under, and a build + // with different numeric widths or restriction flags refuses it *by + // design* — that refusal is what `abi.rs` is for. The fixture is one + // build's bytes, so it can only be checked on that build; anywhere else + // this would be testing the ABI guard rather than the encoding. + Err(err) if format!("{err}").contains("written with") => { + println!("skipped: the golden fixture is a default-build artifact ({err})"); + return; + } + Err(err) => panic!( + "the golden artifact no longer loads: {err}\n\ + The format moved. If that was deliberate, regenerate the fixture with \ + `REGENERATE_GOLDEN=1 cargo test --test format golden`.", + ), + }; + + // A fixture only pins what it contains, and narrowing one while editing the + // source is easy and silent. These are read off the *artifact*, so they say + // what the encoder branch coverage actually is rather than what the source + // looks like it should give. + let kinds: std::collections::BTreeSet = + rhaigrain::bytecode::disassemble(loaded.code()) + .map(|(_, op)| { + format!("{op:?}") + .split(['(', ' ', '{']) + .next() + .unwrap_or_default() + .to_string() + }) + .collect(); + for required in [ + "Chain", // a chain record, with all three of its roots + "CallRef", // and both by-reference call forms + "Rotate", // which only a named receiver needs + "LoadNamed", // the caller's variable, flat + "LoadSharedNamed", // and as the cell a capture binds + "MakeClosure", // a function pointer to a compiled chunk + "Curry", // with what it captured bound onto it + "MakeArray", // a literal the optimizer could not fold + "MakeMap", // and its template-plus-pairs cousin + "CheckSize", // the per-element size check beside it + "PushHandler", // a handler region, whose catch variable is pooled + "Throw", // + "IterNext", // an iterator, and the two-edged instruction + "InterpolateAppend", // a string built a segment at a time + ] { + assert!( + kinds.contains(required), + "the golden no longer contains `{required}`, so its encoder branch \ + is unpinned again — put it back or say why it went", + ); + } + assert!( + kinds.len() >= 35, + "the golden covers only {} instruction kinds, which is narrower than it \ + was written to be: {kinds:?}", + kinds.len(), + ); + + let walked = { + let mut scope = golden_scope(); + let result = engine.eval_ast_with_scope::(&mut scope, &ast); + snapshot(&scope, result) + }; + let ran = { + let mut scope = golden_scope(); + // Whether the program can hand a pointer to a native is read back off + // the bytes, so how it must be run is part of what is being checked. + let result = if loaded.makes_fn_pointers() { + let loaded = loaded.into_shared(); + Vm::new(&engine).run_with_callbacks(&loaded, &mut scope) + } else { + Vm::new(&engine).run(&loaded, &mut scope) + }; + snapshot(&scope, result) + }; + + assert_eq!( + ran, walked, + "the golden artifact no longer means what its source means.\n\ + The format moved without the reader noticing, which is the failure this \ + fixture exists to catch. If the change was deliberate, regenerate with \ + `REGENERATE_GOLDEN=1 cargo test --test format golden`.", + ); + assert!( + ran.result.is_ok(), + "the golden must produce a value, not an error: {ran:?}", + ); +} + +/// A chain rooted at a caller's variable, which the corpus cannot cover. +/// +/// Every case in `writable` runs from an empty scope, so the name a +/// [`Root::Named`] carries — and the position beside it, which is what an +/// `ErrorVariableNotFound` is reported against — has no encoder coverage there. +#[test] +fn a_chain_rooted_at_a_name_survives_the_round_trip() { + let engine = corpus::engine(); + let source = "host.push(2); host[9]"; + + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "the chain must lower"); + let bytes = program.write().expect("must be writable"); + let reloaded = Program::read(&bytes).expect("what we wrote must read back"); + + let seed = |scope: &mut Scope| { + scope.push("host", vec![Dynamic::from(1_i64)]); + }; + + let mut walked = Scope::new(); + seed(&mut walked); + let expected = { + let out = engine.eval_ast_with_scope::(&mut walked, &ast); + snapshot(&walked, out) + }; + + let mut loaded = Scope::new(); + seed(&mut loaded); + let actual = { + let out = Vm::new(&engine).run(&reloaded, &mut loaded); + snapshot(&loaded, out) + }; + + // The mutation lands, and the out-of-bounds index is still blamed on the + // step rather than on the chain — so both the name and its position came + // back intact. + assert_eq!(actual, expected); + assert!(actual.result.is_err(), "the index must still be refused"); +} + +/// Fragments are the allocation the format exists to remove, so writing one +/// would defeat the point. +/// +/// The refusal has to name the construct and where it is. A caller deciding +/// whether to ship source instead cannot act on "27 fragments"; it can act on +/// "for at line 1". +#[test] +fn refusing_to_write_names_the_construct_responsible() { + let engine = corpus::engine(); + + for (source, expected) in [ + ("let x = 1; eval(\"x\")", "an unlowered expression"), + // `?.` short-circuits on unit rather than stepping, so it is not a + // chain this compiler can express whatever its root is. + ("let x = 1; x?.y", "an unlowered expression"), + ] { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert!( + program.residual_count() > 0, + "{source:?} must still fragment, or this test has gone stale", + ); + + let Err(err @ WriteError::HasResiduals { construct, pos, .. }) = program.write() else { + panic!("{source:?} must refuse to write"); + }; + assert_eq!(construct, expected, "for {source:?}"); + assert!(!pos.is_none(), "the refusal must say where: {err}"); + assert!(err.to_string().contains(expected), "{err}"); + } +} + +/// A script function is a chunk like any other, so it crosses the wire with +/// the rest of the program. +#[test] +fn script_functions_survive_the_round_trip() { + let engine = corpus::engine(); + + for source in [ + "fn add(a, b) { a + b } add(2, 3)", + "fn fib(n) { if n < 2 { n } else { fib(n - 1) + fib(n - 2) } } fib(6)", + "fn first() { 1 } fn second(x) { first() + x } second(4)", + // Failing inside a function has to keep rhai's wrapping and position. + "fn bad(x) { x / 0 } bad(1)", + ] { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert!( + !program.functions().is_empty(), + "{source:?} must compile its functions, not leave them to the walker", + ); + + let bytes = program.write().expect("must be writable"); + let reloaded = Program::read(&bytes).expect("must load"); + + assert_eq!( + run(&engine, reloaded), + run_stock(&engine, source), + "{source:?} does not mean the same after a round trip", + ); + } +} + +/// A function the compiler cannot lower stays rhai's, and a program that still +/// depends on rhai's copy cannot be written — silently dropping it would +/// produce an artifact that loads and then cannot find its own function. +#[test] +fn a_function_the_compiler_cannot_lower_refuses_to_write() { + let engine = corpus::engine(); + // `this` is not a scope entry, so no slot addresses it. + let ast = engine + .compile("fn double() { this * 2 } let x = 21; x.double()") + .expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert!( + program.functions().is_empty(), + "a body using `this` must not become a chunk", + ); + assert!( + matches!( + program.write(), + Err(WriteError::HasScriptFunctions | WriteError::HasResiduals { .. }), + ), + "got {:?}", + program.write(), + ); +} + +/// A program with one of everything the corruption tests need to reach: a +/// float constant, a loop, and a `switch` — whose table is the one part of an +/// artifact holding jump targets that are not in the code, and so the one the +/// verifier would most plausibly forget to check. +/// +/// Both by-reference call forms are here too. Their argument count includes a +/// receiver that is not on the operand stack, so a corrupted one is read +/// against a different depth than any other call's. +fn sample(engine: &Engine) -> Vec { + let ast = engine + .compile( + "let a = 1; let b = 2.5; while a < 10 { a += 1 } \ + let c = [a]; push(c, b); push(caller_supplied, a); \ + caller_supplied[0] = a; \ + switch a { 1 => \"one\", 0..=20 => \"some\", _ => \"many\" }", + ) + .expect("must compile"); + Compiler::new() + .compile(&ast) + .write() + .expect("the sample must be writable") +} + +#[test] +fn something_that_is_not_an_artifact_is_refused_at_the_first_bytes() { + assert_eq!(Program::read(b"").unwrap_err(), ReadError::Truncated); + assert_eq!( + Program::read(b"not an artifact at all").unwrap_err(), + ReadError::BadMagic, + ); +} + +#[test] +fn a_future_format_version_is_refused_rather_than_guessed_at() { + let engine = corpus::engine(); + let mut bytes = sample(&engine); + bytes[4] = 0xff; + bytes[5] = 0xff; + + assert!(matches!( + Program::read(&bytes).unwrap_err(), + ReadError::UnsupportedVersion { found: 0xffff, .. }, + )); +} + +/// The fingerprint is the difference between a clean failure and integers +/// decoded as the wrong type, so the error must name the flag. +#[test] +fn a_different_value_representation_is_refused_by_name() { + let engine = corpus::engine(); + + let mut narrow = sample(&engine); + narrow[6] = 4; // INT width + let message = Program::read(&narrow).unwrap_err().to_string(); + assert!( + message.contains("INT") && message.contains('4'), + "the message must name the width: {message}", + ); + + let mut restricted = sample(&engine); + restricted[8] ^= 0b100; // the `no_index` bit + let message = Program::read(&restricted).unwrap_err().to_string(); + assert!( + message.contains("no_index"), + "the message must name the flag: {message}", + ); +} + +/// A `switch` carries hashes rhai's parser computed, and rhai seeds its hasher +/// per process unless the host says otherwise. Two processes that disagree +/// would load each other's artifacts perfectly and then send every subject to +/// the default — a wrong answer rather than a failure, which is the worst kind. +/// +/// The probe is what turns it into a failure, so this checks the failure +/// happens and that the message says what to do about it. +#[test] +fn a_switch_hashed_by_a_different_seed_is_refused() { + let engine = corpus::engine(); + let bytes = sample(&engine); + + // The probe is the only place the artifact repeats this value, and finding + // it that way means the test does not have to know the layout. + let probe = rhaigrain::bytecode::probe().to_le_bytes(); + let at = bytes + .windows(probe.len()) + .position(|window| window == probe) + .expect("an artifact with a switch in it carries a probe"); + + let mut corrupt = bytes.clone(); + corrupt[at] ^= 1; + + let err = Program::read(&corrupt).expect_err("a foreign hasher must be refused"); + assert!( + matches!(err, ReadError::HashSeedMismatch { .. }), + "got {err:?}", + ); + assert!( + err.to_string().contains("set_hashing_seed"), + "the message must say how to fix it: {err}", + ); + + // And the uncorrupted one still loads, so the check is not simply always + // failing. + assert!(Program::read(&bytes).is_ok()); +} + +/// An artifact arrives over a link, so every prefix of one is a thing that can +/// actually turn up. None may load, and none may panic. +#[test] +fn every_truncation_fails_cleanly() { + let engine = corpus::engine(); + let bytes = sample(&engine); + + for cut in 0..bytes.len() { + assert!( + Program::read(&bytes[..cut]).is_err(), + "a {cut}-byte prefix of a {}-byte artifact loaded", + bytes.len(), + ); + } + + assert!(Program::read(&bytes).is_ok(), "the whole thing must load"); +} + +/// Trailing bytes mean the file is not what it says it is, even though every +/// field parsed. Accepting them would let a valid artifact carry a payload. +#[test] +fn trailing_bytes_are_refused() { + let engine = corpus::engine(); + let mut bytes = sample(&engine); + bytes.push(0); + + assert_eq!( + Program::read(&bytes).unwrap_err(), + ReadError::TrailingBytes { count: 1 }, + ); +} + +/// The safety claim in one test: a corrupted artifact is a `Result`, never a +/// panic and never a chunk the VM will touch. +/// +/// Flipping each bit of each byte is exhaustive over single-bit corruption, +/// which is what a bad link produces. Whatever survives has been through the +/// verifier, so it is safe to run — and running it here is what proves the +/// verifier is actually on the load path. +/// +/// **Verification is not termination.** A flipped jump target that still lands +/// inside the chunk is a structurally valid infinite loop, and this test hung +/// until it ran under a budget. That is not a gap to close — no loader can +/// decide halting — it is the reason `Op::Tick` sits on every back edge and the +/// reason the patch exposes `track_operation`. A host running untrusted +/// bytecode must set `max_operations`, exactly as it must for untrusted source. +#[test] +fn no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk() { + let writer = Engine::new(); + let bytes = sample(&writer); + + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let mut loaded = 0usize; + + for index in 0..bytes.len() { + for bit in 0..8 { + let mut corrupt = bytes.clone(); + corrupt[index] ^= 1 << bit; + + if let Ok(program) = Program::read(&corrupt) { + loaded += 1; + program + .verify() + .expect("read must not return a chunk that fails verification"); + // The result is free to be anything; not crashing is the claim. + let _ = Vm::new(&engine).run(&program, &mut Scope::new()); + } + } + } + + // Most flips land in a length, a tag or the fingerprint and are rejected. + // Some land in a constant's value and legitimately still load; that is the + // case worth having run above. + println!( + "{loaded} of {} single-bit corruptions still loaded", + bytes.len() * 8, + ); +} + +/// The whole point of the split, end to end. +/// +/// The device is sent a stripped artifact and knows nothing about the source. +/// It fails, and all it can say is which instruction. The host kept the table, +/// and turns that back into the position rhai itself would have reported. +#[test] +fn a_stripped_program_reports_an_address_the_host_can_resolve() { + let engine = corpus::engine(); + let source = "let a = 1;\nlet b = 0;\na / b"; + + // Host: compile, split. + let ast = engine.compile(source).expect("must compile"); + let full = Compiler::new().compile(&ast); + let expected = run_stock(&engine, source); + let (shipped, table) = full.write_stripped().expect("must be writable"); + + // Device: run bytes, with no table and no source. + let device = Program::read(&shipped).expect("the device must load it"); + assert!( + device.positions().is_stripped(), + "a stripped artifact must not carry positions", + ); + + let mut vm = Vm::new(&engine); + let error = vm + .run(&device, &mut Scope::new()) + .expect_err("dividing by zero must fail"); + let address = vm.fault_pc().expect("a failed run must name an instruction"); + + // Host: resolve what came back. + let site = rhaigrain_pos::resolve(&table, address as u32) + .expect("the failing instruction must have a recorded site"); + + assert_eq!( + (site.line, site.column), + (3, 3), + "the division is at line 3, column 3 of {source:?}", + ); + + // And the same program with its table attached says so itself, exactly as + // rhai does — which is what makes the resolved site trustworthy. + let mut reattached = Program::read(&shipped).unwrap(); + reattached + .attach_positions(&table) + .expect("its own table must attach"); + assert_eq!(run(&engine, reattached), expected); + + // The stripped run is the same failure, minus the position. + assert!( + error.position().is_none(), + "a stripped program has no position to report, got {:?}", + error.position(), + ); +} + +/// Attaching another program's table would misreport every error rather than +/// reporting none, which is strictly worse than having no table. +#[test] +fn a_table_from_a_different_program_is_refused() { + let engine = corpus::engine(); + + let short = Compiler::new().compile(&engine.compile("1 + 1").unwrap()); + let long = Compiler::new() + .compile(&engine.compile("let a = 1; while a < 9 { a += 1 } a").unwrap()); + + let (_, long_table) = long.write_stripped().expect("must be writable"); + let (short_bytes, _) = short.write_stripped().expect("must be writable"); + + let mut program = Program::read(&short_bytes).unwrap(); + assert!( + program.attach_positions(&long_table).is_err(), + "a table naming instructions this chunk does not have must be refused", + ); +} + +/// A stripped artifact that arrives with a table still in it is a contradiction +/// the reader should not paper over. +#[test] +fn an_artifact_carrying_a_mismatched_table_does_not_load() { + let engine = corpus::engine(); + let bytes = sample(&engine); + let program = Program::read(&bytes).expect("the sample must load"); + + assert!( + !program.positions().is_stripped(), + "`write` keeps the table, so this one must have positions", + ); +} + +/// What the split costs, and what it saves. +#[test] +fn stripping_positions_shrinks_the_artifact() { + let engine = corpus::engine(); + + let mut with = 0usize; + let mut without = 0usize; + let mut tables = 0usize; + + for (name, _, full) in writable(&engine) { + let ast = engine.compile(corpus::CASES.iter().find(|c| c.name == name).unwrap().source); + let program = Compiler::new().compile(&ast.unwrap()); + let (stripped, table) = program.write_stripped().expect("must be writable"); + + with += full.len(); + without += stripped.len(); + tables += table.len(); + } + + println!( + "\n{with} bytes with positions -> {without} stripped ({:.0}% smaller), \ + {tables} bytes of table kept behind", + 100.0 * (with - without) as f64 / with as f64, + ); + + assert!( + without < with, + "stripping must actually remove something: {without} vs {with}", + ); +} + +/// The number this project exists to move: bytes retained per source byte, +/// against the 24 a rhai `AST` costs on device. +/// +/// This is the host-side artifact size, not device heap — M6 measures that. +/// What it establishes here is the encoding's own density, which is the part +/// the format controls. +#[test] +fn artifact_size_census() { + let engine = corpus::engine(); + let written = writable(&engine); + + let mut source_bytes = 0usize; + let mut artifact_bytes = 0usize; + let mut rows: Vec<_> = written + .iter() + .map(|(name, source, bytes)| { + source_bytes += source.len(); + artifact_bytes += bytes.len(); + (*name, source.len(), bytes.len()) + }) + .collect(); + + rows.sort_by_key(|(_, _, artifact)| std::cmp::Reverse(*artifact)); + println!("\n{:>7} {:>7} script", "source", "bytes"); + for (name, source, artifact) in &rows { + println!("{source:>7} {artifact:>7} {name}"); + } + println!( + "\n{} scripts: {source_bytes} source bytes -> {artifact_bytes} artifact bytes ({:.2}x)", + rows.len(), + artifact_bytes as f64 / source_bytes as f64, + ); + + // Not a target, a tripwire. The plan is explicit that bytecode need not + // beat minified source on bytes — but an encoding several times larger + // than its input has a bug in it, not a tradeoff. + assert!( + artifact_bytes < source_bytes * 3, + "{artifact_bytes} artifact bytes for {source_bytes} of source is not an encoding", + ); +} diff --git a/tests/grain/fuzz.rs b/tests/grain/fuzz.rs new file mode 100644 index 000000000..4ffe50a8a --- /dev/null +++ b/tests/grain/fuzz.rs @@ -0,0 +1,502 @@ +//! Randomised loading, on the assumption that an artifact is hostile. +//! +//! `Program::read` is the only place untrusted bytes enter, and what it hands +//! back is executed in place — so the claim it has to support is total: any +//! byte string either fails to load or loads into a chunk the VM can run +//! without panicking, looping forever, or reading outside itself. +//! +//! `no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk` in `format.rs` is the +//! exhaustive half, over one-bit corruption of one artifact. This is the wide +//! half: many mutations, several of them structural, over a corpus of real +//! artifacts. Seeded rather than random, so a failure is reproducible and CI +//! does not go intermittent — the seed is printed and any interesting input is +//! dumped as hex. +//! +//! It is not a substitute for `cargo fuzz` (see `fuzz/`), which explores with +//! coverage feedback. It is what runs on every `cargo test`. + +// Only the sources are wanted here; the names belong to the harnesses that +// report per-case results. +#[allow(dead_code)] +mod corpus; + +use rhai::{Dynamic, Engine, Scope}; +use rhaigrain::{Compiler, Program, Vm}; + +use corpus::generate::{Generator, Rng}; + +/// Ways an artifact can arrive wrong. Truncation and splicing matter as much +/// as corruption: a length field that disagrees with what follows is how a +/// loader is talked into reading past the end. +fn mutate(rng: &mut Rng, bytes: &[u8]) -> Vec { + let mut out = bytes.to_vec(); + if out.is_empty() { + return out; + } + + match rng.below(6) { + // Corrupt a run of bytes, which reaches multi-byte fields that + // single-bit flipping cannot. + 0 | 1 => { + let at = rng.below(out.len()); + let run = 1 + rng.below((out.len() - at).min(8)); + for byte in &mut out[at..at + run] { + *byte = rng.next() as u8; + } + } + // Cut it short. + 2 => out.truncate(rng.below(out.len())), + // Splice out a slice, so every following offset is wrong. + 3 => { + let at = rng.below(out.len()); + let run = 1 + rng.below((out.len() - at).min(16)); + out.drain(at..at + run); + } + // Insert junk, the same problem in the other direction. + 4 => { + let at = rng.below(out.len()); + let junk: Vec = (0..1 + rng.below(16)).map(|_| rng.next() as u8).collect(); + out.splice(at..at, junk); + } + // Set a byte to an edge value: the lengths and counts are varints, and + // 0xff runs are what make one disagree with reality. + _ => { + let at = rng.below(out.len()); + out[at] = [0x00, 0x01, 0x7f, 0x80, 0xff][rng.below(5)]; + } + } + out +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// The claim, over a lot of inputs. +/// +/// Running what survives is the point rather than a bonus: a loader that +/// accepts a chunk it should not have has done nothing visible until something +/// executes it. +#[test] +fn mutated_artifacts_load_or_fail_but_never_misbehave() { + let writer = corpus::engine(); + + // Every artifact the corpus can produce, so the mutations land on real + // structure — a hand-written sample would only ever exercise its own + // shape. + let seeds: Vec> = corpus::CASES + .iter() + .filter_map(|case| { + let ast = writer.compile(case.source).ok()?; + Compiler::new().compile(&ast).write().ok() + }) + .collect(); + + assert!( + seeds.len() >= 50, + "only {} artifacts to mutate, which is too few to prove anything", + seeds.len(), + ); + + // A budget, because verification proves structure and not termination: a + // corrupted-but-in-range jump target is a valid infinite loop. + let mut engine = corpus::engine(); + engine.set_max_operations(10_000); + engine.set_max_string_size(4096); + engine.set_max_array_size(1024); + + const SEED: u64 = 0x5eed_1234_abcd_0001; + // Enough to be worth running on every `cargo test` and not so many that + // anyone is tempted to stop. Depth is `cargo fuzz`'s job — see `fuzz/`. + const ROUNDS: usize = 20; + + let mut rng = Rng::new(SEED); + let mut loaded = 0usize; + + for round in 0..ROUNDS { + for original in &seeds { + let corrupt = mutate(&mut rng, original); + + let Ok(program) = Program::read(&corrupt) else { + continue; + }; + loaded += 1; + + // Anything `read` returns must already have verified — that is the + // contract the VM's missing bounds checks rest on. + assert!( + program.verify().is_ok(), + "seed {SEED:#x} round {round}: read returned a chunk that does \ + not verify: {}", + hex(&corrupt), + ); + + // And must then run without taking the process down. The result is + // free to be anything at all. + let _ = Vm::new(&engine).run(&program, &mut Scope::new()); + } + } + + println!( + "{loaded} of {} mutations loaded and ran", + ROUNDS * seeds.len(), + ); + assert!( + loaded > 0, + "no mutation survived, so nothing was actually executed", + ); +} + +/// Bytes that were never an artifact, which is the other way in. +#[test] +fn arbitrary_bytes_never_load_into_something_that_misbehaves() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let mut rng = Rng::new(0x5eed_0000_0000_0002); + + for _ in 0..20_000 { + let len = rng.below(64); + let mut bytes: Vec = (0..len).map(|_| rng.next() as u8).collect(); + + // Half of them get the magic, so the loader is reached rather than + // rejected at the first four bytes every time. + if rng.below(2) == 0 && bytes.len() >= 4 { + bytes[..4].copy_from_slice(b"RGRN"); + } + + if let Ok(program) = Program::read(&bytes) { + assert!(program.verify().is_ok(), "unverified: {}", hex(&bytes)); + let _ = Vm::new(&engine).run(&program, &mut Scope::new()); + } + } +} + +/// What a script produced, in a form two runs can be compared on. +#[derive(PartialEq, Eq)] +struct Outcome { + result: Result, + scope: Vec<(String, String)>, +} + +impl std::fmt::Debug for Outcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.result { + Ok(value) => write!(f, "{value} | scope {:?}", self.scope), + Err(err) => write!(f, "!{err} | scope {:?}", self.scope), + } + } +} + +fn snapshot(scope: &Scope, result: Result>) -> Outcome { + Outcome { + result: result + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")), + scope: scope + .iter_raw() + .map(|(name, _, value)| (name.to_string(), format!("{value:?}"))) + .collect(), + } +} + +/// Whether an outcome is one the two sides are not expected to reach together. +/// +/// Both enforce these limits; neither counts towards them in lockstep, because +/// a VM instruction is not a walker node and a callback boundary costs a +/// different number of call levels. A script that runs into one has stopped +/// saying anything about lowering, so it is dropped rather than compared — +/// and the drop rate is asserted, so this cannot quietly become the answer for +/// everything. +fn hit_a_limit(outcome: &Outcome) -> bool { + let Err(err) = &outcome.result else { + return false; + }; + ["ErrorTooManyOperations", "ErrorStackOverflow", "ErrorTooManyVariables"] + .iter() + .any(|limit| err.contains(limit)) +} + +/// Run something that may panic, without the panic reaching the console. +/// +/// The hook is swapped rather than left alone because a fuzzing run can trip +/// the same upstream panic thousands of times, and the backtrace for each one +/// is noise. Single-threaded, and restored immediately. +fn quietly(body: impl FnOnce() -> T) -> Option { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(body)).ok(); + std::panic::set_hook(hook); + out +} + +/// A rhai bug, not one of ours, found by `cargo fuzz run generated`. +/// +/// A `switch` as the last statement of a block makes rhai's optimizer delete a +/// `let` in that block and flatten what is left into the enclosing statement +/// list — while the reads of that local keep the scope index the parser gave +/// them. The index is counted back from the end of the scope, so it now names +/// whatever moved into its place. +/// +/// Two symptoms, and the quiet one is the dangerous one: +/// +/// * with something else at that index, rhai answers with **another variable's +/// value** and reports nothing at all; +/// * with nothing there, `scope.len() - index` underflows (`eval/expr.rs:131`) +/// — a panic in a debug build and a wild index in a release one. +/// +/// Both are three lines of ordinary rhai with none of this involved. We resolve +/// locals by name, so we say the variable is missing, which is the closest +/// thing to right available: there is no agreeing with an AST that refers to a +/// local it does not declare. +/// +/// Pinned because `generated_scripts_agree_with_the_walker` and both `cargo +/// fuzz` targets have to skip these, and every one of those skips should go the +/// day this test starts failing. +#[test] +fn rhai_drops_a_local_its_optimizer_still_refers_to() { + // The read lands on `a`, so rhai answers 1 where the script says 99. + const WRONG: &str = "let a = 1; { let b = 99; switch b { _ => b } }"; + // The same shape with nothing left at that index. + const PANIC: &str = "{ let b = 99; switch b { _ => b } }"; + + let engine = corpus::engine(); + let mut plain = corpus::engine(); + plain.set_optimization_level(rhai::OptimizationLevel::None); + + let ast = engine.compile(WRONG).expect("it parses"); + let value = engine + .eval_ast::(&ast) + .expect("rhai runs it, which is the problem"); + assert_eq!( + value.as_int().ok(), + Some(1), + "rhai no longer reads the wrong variable — delete the optimizer skips \ + in the fuzzers and this test with them", + ); + + let ast = engine.compile(PANIC).expect("it parses"); + assert!( + quietly(|| engine.eval_ast::(&ast)).is_none(), + "rhai no longer underflows here — delete the walker skip in \ + `generated_scripts_agree_with_the_walker`", + ); + + // The same scripts with the optimizer out of the way, which is what says + // the fault is in the optimizer rather than in them. + for source in [WRONG, PANIC] { + let ast = plain.compile(source).expect("it parses either way"); + let value = quietly(|| plain.eval_ast::(&ast)) + .expect("with the optimizer off it should run") + .expect("and it should not fail"); + assert_eq!(value.as_int().ok(), Some(99), "{source:?}"); + } +} + +/// Whether the two sides agree once rhai's optimizer is out of the way. +/// +/// The optimizer is what makes a dropped local reachable, so agreeing without +/// it and disagreeing with it says the AST is at fault rather than the +/// lowering. Only ever asked about a divergence that already looks like one. +fn agree_unoptimised(source: &str) -> bool { + let mut plain = corpus::engine(); + plain.set_optimization_level(rhai::OptimizationLevel::None); + let Ok(ast) = plain.compile(source) else { + return false; + }; + + let mut walker_scope = Scope::new(); + let Some(walked) = quietly(|| plain.eval_ast_with_scope::(&mut walker_scope, &ast)) + else { + return false; + }; + + let program = Compiler::new().compile(&ast); + let mut vm_scope = Scope::new(); + let ours = if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(&plain).run_with_callbacks(&program, &mut vm_scope) + } else { + Vm::new(&plain).run(&program, &mut vm_scope) + }; + + snapshot(&walker_scope, walked) == snapshot(&vm_scope, ours) +} + +/// The claim the corpus makes, over scripts nobody wrote. +/// +/// `tests/differential.rs` pins the constructs someone thought of. This is the +/// same comparison — value, error variant, error position, and what was left in +/// the scope — over combinations of them that no one did. +#[test] +fn generated_scripts_agree_with_the_walker() { + // Budgets, not correctness limits: a generated script can be quadratic in + // a way no corpus case is, and there is no reason to wait for it. + let mut engine = corpus::engine(); + engine.set_max_operations(200_000); + engine.set_max_array_size(2048); + engine.set_max_string_size(8192); + // Pinned, because rhai's defaults for these are `debug_assertions`-gated — + // 32/16 and 8 in a debug build against 64/64 and 64 in a release one + // (`api/limits.rs:10-36`). Left alone, `cargo test` and + // `cargo test --release` parse different halves of the same seeded corpus + // and compare different scripts, which is not a thing a differential + // harness may do. The release numbers, because they admit more. + engine.set_max_expr_depths(64, 64); + engine.set_max_call_levels(64); + + const SEED: u64 = 0x5eed_9a11_0000_0001; + const SCRIPTS: usize = 4000; + + let mut parsed = 0usize; + let mut ran = 0usize; + let mut valued = 0usize; + let mut skipped = 0usize; + let mut walker_panics = 0usize; + let mut too_deep = 0usize; + let mut lost_a_local = 0usize; + let mut unparsed: Vec = Vec::new(); + let mut failures = Vec::new(); + + for n in 0..SCRIPTS { + // Seeded per script rather than from one long stream, so a failing + // script can be reproduced on its own by its index. + let source = Generator::new(SEED ^ n as u64).script(); + + let ast = match engine.compile(&source) { + Ok(ast) => ast, + Err(err) => { + if matches!(err.err_type(), rhai::ParseErrorType::ExprTooDeep) { + too_deep += 1; + } else { + unparsed.push(format!("\n {err}\n {source}")); + } + continue; + } + }; + parsed += 1; + + // The scope is snapshotted after the run, not during: what a script + // leaves behind is half of what is being compared. + // + // Guarded because the walker can panic on a script of its own accord — + // see `rhai_underflows_a_scope_index_under_its_own_optimizer`. There is + // nothing to compare against a side that did not finish, so those are + // counted and dropped rather than blamed on the VM. + let mut walker_scope = Scope::new(); + let Some(walked) = quietly(|| engine.eval_ast_with_scope::(&mut walker_scope, &ast)) + else { + walker_panics += 1; + continue; + }; + let walked = snapshot(&walker_scope, walked); + + if hit_a_limit(&walked) { + skipped += 1; + continue; + } + ran += 1; + if walked.result.is_ok() { + valued += 1; + } + + let program = Compiler::new().compile(&ast); + let mut vm_scope = Scope::new(); + let ours = if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(&engine).run_with_callbacks(&program, &mut vm_scope) + } else { + Vm::new(&engine).run(&program, &mut vm_scope) + }; + let ours = snapshot(&vm_scope, ours); + + if hit_a_limit(&ours) { + skipped += 1; + continue; + } + + if walked == ours { + continue; + } + // Not every disagreement is one to have: rhai's optimizer can delete a + // `let` whose variable is still read, and then there is no agreeing + // with it. See `rhai_drops_a_local_its_optimizer_still_refers_to`. + if format!("{ours:?}").contains("ErrorVariableNotFound") + && agree_unoptimised(&source) + { + lost_a_local += 1; + continue; + } + + if failures.len() < 5 { + failures.push(format!( + "\n=== script {n} (seed {:#x}) ===\n {source}\n rhai: {walked:?}\n vm: {ours:?}", + SEED ^ n as u64, + )); + } + } + + println!( + "{parsed} of {SCRIPTS} generated scripts parsed, {ran} compared, \ + {valued} produced a value, {skipped} hit a limit, \ + {walker_panics} panicked the walker, \ + {too_deep} were too complex to parse, \ + {lost_a_local} lost a local to rhai's optimizer", + ); + + // The walker panicking is upstream's problem, but it is also a hole in this + // test's coverage, so it is worth knowing if it ever becomes common. + assert!( + walker_panics * 50 < SCRIPTS, + "{walker_panics} of {SCRIPTS} scripts panicked the walker, which is too \ + many to keep skipping — see \ + `rhai_underflows_a_scope_index_under_its_own_optimizer`", + ); + + // A generator that mostly emits garbage would pass this test by comparing + // two identical parse failures a few thousand times. These are the numbers + // that say it is doing work, and they are what to look at first if this + // file ever stops finding anything. + // + // Invalid *syntax* is the thing to catch, and a raw parse rate does not + // catch it: a script rejected for exceeding rhai's complexity limit is + // well-formed, and how many do is a property of how much the generator + // packs into one script rather than of whether it can write the language. + // So the two are counted apart, and this is the one that means something. + assert!( + unparsed.is_empty(), + "{} of {SCRIPTS} scripts are not valid rhai, so the generator is \ + emitting syntax rather than testing it:{}", + unparsed.len(), + unparsed.iter().take(5).cloned().collect::(), + ); + assert!( + parsed * 10 >= SCRIPTS * 7, + "only {parsed} of {SCRIPTS} scripts parsed, and {too_deep} were rejected \ + as too complex — the generator is packing more into a script than rhai \ + will take, so most of what it writes is never run", + ); + // Against the whole corpus rather than against what ran, because the two + // move for opposite reasons. A script that errors is still compared — the + // variant, the position and the scope all have to match — so a *rising* + // error rate is usually the generator reaching further, and measuring + // against `ran` would read that as a regression and push it back toward + // scripts too simple to catch anything. What this is here to catch is the + // generator producing nothing that works at all. + assert!( + valued * 5 >= SCRIPTS, + "only {valued} of {SCRIPTS} scripts reached a value; the rest failed at \ + run time, which tests error parity and little else", + ); + assert!( + skipped * 20 < SCRIPTS, + "{skipped} of {SCRIPTS} scripts hit an engine limit, so the budgets are \ + deciding the outcome rather than the code", + ); + + assert!( + failures.is_empty(), + "generated scripts diverged:{}", + failures.join(""), + ); +} diff --git a/tests/grain/limits.rs b/tests/grain/limits.rs new file mode 100644 index 000000000..19bab1c79 --- /dev/null +++ b/tests/grain/limits.rs @@ -0,0 +1,219 @@ +//! Compiled code must still be stoppable. +//! +//! Rhai enforces `max_operations` and the `on_progress` interrupt from +//! `Engine::track_operation`, which the tree walker calls per AST node. A VM +//! that never called it would turn `loop {}` from a script the engine +//! terminates into one that hangs the host — a safety regression, not a +//! performance one, which is why `track_operation` is in the patch. +//! +//! These live outside the differential corpus on purpose. The walker ticks per +//! node and the VM ticks per loop back-edge, so the operation *counts* differ +//! and always will. What must hold is that the limit fires and the interrupt is +//! honoured, so that is what is asserted — not parity of counts or positions. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use rhai::{Dynamic, Engine, EvalAltResult, Scope}; +use rhaigrain::bytecode::Op; +use rhaigrain::{Compiler, Program, Vm}; + +/// A bare infinite loop, which the compiler lowers with nothing left over — +/// asserted below, so this cannot silently become a test of the fallback. +const SPIN: &str = "loop { }"; + +fn run_vm(engine: &Engine, source: &str) -> Result> { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert_eq!( + program.residual_count(), + 0, + "{source:?} must be fully lowered, or this tests rhai rather than the VM", + ); + + // Without a tick on the back-edge nothing in a compiled loop ever reaches + // `track_operation`, and the tests below would hang rather than fail. + assert!( + program + .main() + .ops(program.code()) + .any(|(_, op)| op == Op::Tick), + "{source:?} lowered to a loop with no operation tick", + ); + + Vm::new(engine).run(&program, &mut Scope::new()) +} + +#[test] +fn compiled_loop_hits_the_operation_limit() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let err = run_vm(&engine, SPIN).expect_err("an unbounded loop must be stopped"); + + assert!( + matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), + "expected ErrorTooManyOperations, got {err:?}", + ); +} + +#[test] +fn compiled_loop_honours_the_progress_interrupt() { + let ticks = Arc::new(AtomicU64::new(0)); + let seen = ticks.clone(); + + let mut engine = Engine::new(); + engine.on_progress(move |count| { + seen.store(count, Ordering::SeqCst); + // Stand-in for a host's abort flag. + (count >= 500).then(|| Dynamic::from("terminated")) + }); + + let err = run_vm(&engine, SPIN).expect_err("the interrupt must stop the loop"); + + assert!( + matches!(*err, EvalAltResult::ErrorTerminated(..)), + "expected ErrorTerminated, got {err:?}", + ); + assert!( + ticks.load(Ordering::SeqCst) >= 500, + "on_progress should have been called on every back-edge", + ); +} + +/// A chunk that loops with no tick in it must still be stopped. +/// +/// Every loop this compiler emits carries an `Op::Tick` on its back-edge, so +/// nothing it produces can spin. An artifact is not required to have come from +/// it. Turning this program's tick into a no-op leaves a chunk that still +/// verifies — the jump is in range, the stack balances, every path reaches a +/// `Return` — and runs forever, which makes the engine's budget the only thing +/// between a host and a hostile file. +/// +/// So the budget cannot depend on the compiler having been generous: the VM +/// charges an operation for every *backward* transfer, and a cycle always has +/// one. Found by `mutated_artifacts_load_or_fail_but_never_misbehave`, which +/// hung on a mutation rather than failing. +#[test] +fn a_loop_with_its_tick_removed_still_hits_the_limit() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let ast = engine.compile(SPIN).expect("must compile"); + let program = Compiler::new().compile(&ast); + + // Where the tick sits inside the code, and what the code looks like, so the + // same bytes can be found again inside the finished artifact. + let code = program.code().to_vec(); + let (tick_at, _) = program + .main() + .ops(program.code()) + .find(|(_, op)| *op == Op::Tick) + .expect("the compiler ticks a loop"); + + let mut bytes = program.write().expect("a lowered program must write"); + let start = bytes + .windows(code.len()) + .position(|window| window == code) + .expect("the artifact embeds the code verbatim"); + + // `Checkpoint` is the other one-byte instruction that does nothing to the + // stack, so this swap leaves every offset, jump target and position entry + // exactly where it was. Only the metering goes. + bytes[start + tick_at] = rhaigrain::bytecode::code::tag::CHECKPOINT; + + let tickless = Program::read(&bytes).expect("still a valid artifact"); + assert!( + !tickless + .main() + .ops(tickless.code()) + .any(|(_, op)| op == Op::Tick), + "the tick should be gone, or this tests nothing", + ); + + let err = Vm::new(&engine) + .run(&tickless, &mut Scope::new()) + .expect_err("a tickless loop must still be stopped"); + assert!( + matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), + "expected ErrorTooManyOperations, got {err:?}", + ); +} + +/// The walker and the VM must agree that the script *fails*, even though they +/// disagree about after how many operations. +#[test] +fn the_walker_agrees_the_loop_is_stopped() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let ast = engine.compile(SPIN).expect("must compile"); + let err = engine + .eval_ast_with_scope::(&mut Scope::new(), &ast) + .expect_err("rhai must stop it too"); + + assert!( + matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), + "expected ErrorTooManyOperations, got {err:?}", + ); +} + +/// `max_string_size` is a host's defence, and interpolation is the easiest way +/// to walk past it — rhai checks the running total after *every* segment +/// rather than once at the end, so a script cannot build a huge string and +/// hand it over. +/// +/// The position is checked too, because it is the one thing a single +/// instruction might not be able to reproduce: rhai blames the segment that +/// tipped the total over, and the VM has one position-table entry per +/// instruction. +#[test] +fn interpolation_respects_the_string_limit() { + let mut engine = Engine::new(); + engine.set_max_string_size(16); + + let source = r#"let a = "0123456789"; `${a}${a}${a}`"#; + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "must be lowered, not walked"); + + let walker = engine + .eval_ast_with_scope::(&mut Scope::new(), &ast) + .expect_err("the walker must refuse it"); + let vm = Vm::new(&engine) + .run(&program, &mut Scope::new()) + .expect_err("and so must the VM"); + + assert!( + matches!(*vm, EvalAltResult::ErrorDataTooLarge(..)), + "got {vm:?}", + ); + assert_eq!( + format!("{vm:?}"), + format!("{walker:?}"), + "including the position of the segment that went over", + ); +} + +/// A loop that does terminate must not be killed by the tick itself, and must +/// still produce the value rhai produces. +#[test] +fn ticking_does_not_disturb_a_bounded_loop() { + let mut engine = Engine::new(); + engine.set_max_operations(10_000); + + let source = "let i = 0; loop { i += 1; if i > 100 { break i; } }"; + let ast = engine.compile(source).expect("must compile"); + + let program = Compiler::new().compile(&ast); + let vm = Vm::new(&engine) + .run(&program, &mut Scope::new()) + .expect("bounded loop must finish"); + + let walker = engine + .eval_ast_with_scope::(&mut Scope::new(), &ast) + .expect("bounded loop must finish under rhai too"); + + assert_eq!(format!("{vm:?}"), format!("{walker:?}")); +} diff --git a/tests/grain/patch_integrity.rs b/tests/grain/patch_integrity.rs new file mode 100644 index 000000000..dcc3bd0f4 --- /dev/null +++ b/tests/grain/patch_integrity.rs @@ -0,0 +1,32 @@ +//! The fork must stay reviewable. +//! +//! rhaigrain builds against a vendored rhai rather than the published crate, so +//! that the VM can reach a handful of internals rhai does not export. That is +//! only defensible while the delta is small, visibility-only, and visible in +//! one file. `vendor/rhai` is generated and gitignored; `vendor/rhai-1.25.1.patch` +//! is the checked-in artifact, and this test is what keeps them honest. +//! +//! The real work is in `scripts/check-patch.sh`, which verifies two separate +//! claims: that the patch equals the true pristine -> vendored delta, and that +//! applying it to a pristine tree reproduces what we compile against. + +use std::process::Command; + +#[test] +fn vendored_rhai_matches_its_patch() { + let root = env!("CARGO_MANIFEST_DIR"); + + let output = Command::new("bash") + .arg(format!("{root}/scripts/check-patch.sh")) + .output() + .expect("failed to run scripts/check-patch.sh"); + + if !output.status.success() { + panic!( + "vendored rhai does not match its patch\n\ + --- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } +} diff --git a/tests/grain/projection.rs b/tests/grain/projection.rs new file mode 100644 index 000000000..4152b2620 --- /dev/null +++ b/tests/grain/projection.rs @@ -0,0 +1,242 @@ +//! What would a lowering of this script actually weigh? +//! +//! M1's job is to test the premise before building an instruction set to serve +//! it. `follow.rhai` turns out to use most of the language — nested index +//! writes, property chains, method calls, script functions, `loop`/`while`/ +//! `if-else` — so lowering it for real is several milestones of work. Doing +//! that first and *then* discovering the artifact is only marginally smaller +//! than the tree would be the expensive way to learn it. +//! +//! So this counts the real AST, node by node, and prices each node against the +//! planned stack encoding. The per-node costs are stated in `encoded_size` +//! where they can be argued with. +//! +//! **This is a projection, not a measurement.** It is a lower bound in one +//! direction and optimistic in another: it assumes every node lowers to the +//! ops listed, and it ignores the operand stack traffic a real lowering emits +//! for temporaries. The measured number replaces it when M2-M4 land. + +use std::collections::BTreeSet; + +use rhai::{ASTNode, Engine, Expr, Stmt, AST}; + +const SOURCE: &str = include_str!("fixtures/follow.rhai"); + +/// Marks a node kind the model does not know how to price. Its presence is a +/// test failure: a silently-unpriced variant would understate the projection. +const UNPRICED: &str = "UNPRICED"; + +/// A node's kind, and what the planned encoding would spend on it. +/// +/// The encoding is a stack machine: one byte of opcode, then operands as +/// varints that are one byte at this program's scale (fewer than 128 locals, +/// constants or names). Costs below are in bytes. +fn encoded_size(node: &ASTNode) -> (&'static str, usize) { + match node { + ASTNode::Expr(expr) => match expr { + // Small integers get a dedicated opcode with an inline operand; + // anything else is an index into the constant pool. Either way two + // bytes, which is why they are not split here. + Expr::IntegerConstant(..) => ("int", 2), + Expr::FloatConstant(..) => ("float", 2), + Expr::StringConstant(..) => ("string", 2), + Expr::CharConstant(..) => ("char", 2), + Expr::DynamicConstant(..) => ("dynamic const", 2), + // No operand: the opcode is the value. + Expr::BoolConstant(..) => ("bool", 1), + Expr::Unit(..) => ("unit", 1), + Expr::ThisPtr(..) => ("this", 1), + + // LoadLocal + slot. The parser has already resolved most of these + // to a slot index, so this is not an optimistic assumption. + Expr::Variable(..) => ("variable", 2), + + // Operators become a typed opcode with no operands when both sides + // are primitives, and a generic call otherwise. Scored as a call, + // the pessimistic reading, because the deopt guard may disable the + // fast path for a given engine. + Expr::FnCall(x, ..) if x.op_token.is_some() => ("operator", 1), + Expr::FnCall(..) => ("call", 3), + Expr::MethodCall(..) => ("method call", 3), + + // Get by interned name; set is a separate op emitted by the parent. + Expr::Property(..) => ("property", 2), + Expr::Dot(..) => ("dot", 1), + Expr::Index(..) => ("index", 1), + + // Length-prefixed build from the operand stack. + Expr::Array(..) => ("array literal", 2), + Expr::Map(..) => ("map literal", 2), + Expr::InterpolatedString(..) => ("interpolation", 2), + + // Short-circuit: a conditional jump per operand. + Expr::And(..) => ("and", 3), + Expr::Or(..) => ("or", 3), + Expr::Coalesce(..) => ("coalesce", 3), + + // Block-as-expression needs no instruction of its own; the + // statements inside are counted separately. + Expr::Stmt(..) => ("block expr", 0), + + // Cannot be lowered at all — the handler is looked up by string + // against a live Engine. Priced as the residual it would stay. + Expr::Custom(..) => ("custom syntax", 3), + + // `Expr` is #[non_exhaustive]. A variant added by a rhai upgrade + // must surface as a test failure, not get quietly priced at zero. + _ => (UNPRICED, 0), + }, + + ASTNode::Stmt(stmt) => match stmt { + Stmt::Noop(..) => ("noop", 0), + // The block's own statements are counted as they are walked. + Stmt::Block(..) => ("block", 0), + + // StoreLocal + slot. + Stmt::Var(..) => ("var decl", 2), + // Store, plus the operator when compound. + Stmt::Assignment(..) => ("assignment", 3), + + // Condition jump plus the jump over the else arm. + Stmt::If(..) => ("if", 6), + // Condition jump plus the backward jump. + Stmt::While(..) => ("while", 6), + Stmt::Do(..) => ("do", 6), + // Iterator setup, plus a step-and-branch per turn. + Stmt::For(..) => ("for", 8), + // Jump table plus a linear range list. + Stmt::Switch(..) => ("switch", 12), + + // Jumps to a fixed target. + Stmt::BreakLoop(..) => ("break/continue", 3), + Stmt::Return(..) => ("return/throw", 3), + + // Handler region, registered rather than executed. + Stmt::TryCatch(..) => ("try/catch", 8), + + Stmt::FnCall(..) => ("call stmt", 3), + // Discard the value the expression left behind. + Stmt::Expr(..) => ("expr stmt", 1), + + #[cfg(not(feature = "no_module"))] + Stmt::Import(..) => ("import", 3), + #[cfg(not(feature = "no_module"))] + Stmt::Export(..) => ("export", 3), + #[cfg(not(feature = "no_closure"))] + Stmt::Share(..) => ("share", 2), + + _ => (UNPRICED, 0), + }, + + _ => (UNPRICED, 0), + } +} + +/// Names that would live in the artifact's string table, deduplicated. +/// +/// Only the ones reachable without unpacking every boxed payload: variables, +/// called functions and properties. That undercounts, so the projected string +/// table is a floor. +fn interned_names(ast: &AST) -> BTreeSet { + let mut names = BTreeSet::new(); + + ast.walk(&mut |path: &[ASTNode]| { + if let Some(ASTNode::Expr(expr)) = path.last() { + match expr { + Expr::Variable(x, ..) => { + names.insert(x.1.to_string()); + } + Expr::FnCall(x, ..) | Expr::MethodCall(x, ..) => { + names.insert(x.name.to_string()); + } + Expr::StringConstant(s, ..) => { + names.insert(s.to_string()); + } + _ => {} + } + } + true + }); + + names +} + +#[test] +fn projected_artifact_size() { + let engine = Engine::new(); + let ast = engine + .compile(SOURCE) + .expect("fixtures/follow.rhai must compile"); + + let mut counts: std::collections::BTreeMap<&'static str, (usize, usize)> = + std::collections::BTreeMap::new(); + let mut code_bytes = 0usize; + let mut nodes = 0usize; + + ast.walk(&mut |path: &[ASTNode]| { + let Some(node) = path.last() else { + return true; + }; + let (kind, size) = encoded_size(node); + let entry = counts.entry(kind).or_default(); + entry.0 += 1; + entry.1 += size; + code_bytes += size; + nodes += 1; + true + }); + + let names = interned_names(&ast); + // One length byte plus the UTF-8 bytes, per distinct name. + let string_table: usize = names.iter().map(|n| 1 + n.len()).sum(); + + // A tag byte plus eight bytes of payload, for every node that referenced + // the pool. Deduplication would shrink this; not modelling it keeps the + // projection on the pessimistic side. + let pool_refs: usize = ["int", "float", "string", "char", "dynamic const"] + .iter() + .filter_map(|k| counts.get(k)) + .map(|(n, _)| *n) + .sum(); + let const_pool = pool_refs * 9; + + // Header, ABI fingerprint, and section offsets. + const HEADER: usize = 64; + + let projected = HEADER + string_table + const_pool + code_bytes; + let source_bytes = SOURCE.len(); + + println!("\nfixtures/follow.rhai — {source_bytes} source bytes, {nodes} AST nodes\n"); + println!("{:<18} {:>7} {:>9}", "node kind", "count", "bytes"); + for (kind, (n, bytes)) in &counts { + println!("{kind:<18} {n:>7} {bytes:>9}"); + } + + println!("\n{:<18} {:>9}", "code", code_bytes); + println!( + "{:<18} {:>9} ({} distinct names)", + "string table", + string_table, + names.len() + ); + println!("{:<18} {:>9} ({pool_refs} pool refs)", "constants", const_pool); + println!("{:<18} {:>9}", "header", HEADER); + println!("{:<18} {:>9}", "projected total", projected); + + println!( + "\nprojected artifact / source {:.2}x", + projected as f64 / source_bytes as f64 + ); + + assert!(nodes > 0, "the walk visited nothing"); + assert!( + code_bytes > 0, + "every node priced at zero means the model is broken", + ); + assert!( + !counts.contains_key(UNPRICED), + "{} nodes have no cost in the model, so the projection understates: \ + a rhai upgrade added an AST variant", + counts[UNPRICED].0, + ); +} diff --git a/tests/grain/scope.rs b/tests/grain/scope.rs new file mode 100644 index 000000000..09826e30d --- /dev/null +++ b/tests/grain/scope.rs @@ -0,0 +1,638 @@ +//! Variables the caller already had, which no slot can name. +//! +//! Slots are indices into the caller's `Scope` counted from a base taken when +//! the program starts, so everything the host put there beforehand sits below +//! every slot. Those reads and writes go by name instead, and this is where +//! that is held to rhai's behaviour — including the three places rhai looks +//! and the order it looks in. +//! +//! It matters more than the node count suggests: a script that reads anything +//! its host supplied used to be a fragment, and a program with a fragment in +//! it cannot be written to an artifact at all. + +// `on_var` carries rhai's "volatile, may change" marker rather than a real +// deprecation — the same one `eval_expression_tree_raw` carries. Registering a +// resolver is the only way to test that the VM consults one. +#![allow(deprecated)] + +// Only the engine is wanted here; the corpus scripts belong to the harnesses +// that run all of them. +#[allow(dead_code)] +mod corpus; + +use rhai::{Dynamic, Engine, Module, Scope}; +use rhaigrain::{Compiler, Vm}; + +/// What a run produced, in a form two runs can be compared on. +#[derive(Debug, PartialEq, Eq)] +struct Outcome { + result: Result, + scope: Vec<(String, String)>, +} + +fn capture(scope: &Scope, result: Result>) -> Outcome { + Outcome { + result: result + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")), + scope: scope + .iter_raw() + .map(|(name, _, value)| (name.to_string(), format!("{value:?}"))) + .collect(), + } +} + +/// Run `source` under rhai and under the VM, from the same starting scope, and +/// require they agree on the value, the error and what the scope holds after. +/// +/// `writable` is the point of the exercise rather than a detail: a program +/// that still fragments cannot cross a wire, so a case that passes while +/// fragmenting has proved nothing about the feature. +#[track_caller] +fn agree(source: &str, build: impl Fn(&mut Scope), writable: bool) { + agree_with(&corpus::engine(), source, build, writable); +} + +/// The same, against an engine the caller has set up — a variable resolver, a +/// published module. Both change what a name resolves to, which is most of what +/// this file is about. +#[track_caller] +fn agree_with(engine: &Engine, source: &str, build: impl Fn(&mut Scope), writable: bool) { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert_eq!( + program.residual_count() == 0, + writable, + "{source:?} fragments: {:?}", + program.first_unsupported(), + ); + + let mut walked = Scope::new(); + build(&mut walked); + let expected = capture( + &walked.clone(), + engine.eval_ast_with_scope::(&mut walked, &ast), + ); + let expected = Outcome { + scope: capture(&walked, Ok(Dynamic::UNIT)).scope, + ..expected + }; + + let mut run = Scope::new(); + build(&mut run); + let actual = { + let result = Vm::new(engine).run(&program, &mut run); + capture(&run, result) + }; + + assert_eq!(actual, expected, "{source:?}"); +} + +fn lit(value: i64) -> Dynamic { + Dynamic::from(value) +} + +#[test] +fn a_caller_variable_can_be_read() { + agree("brightness * 2", |s| { s.push("brightness", 21_i64); }, true); + agree( + "mode + \"!\"", + |s| { + s.push("mode", "chase".to_string()); + }, + true, + ); +} + +#[test] +fn a_caller_variable_can_be_written() { + agree("brightness = 7; brightness", |s| { s.push("brightness", 1_i64); }, true); + agree("brightness += 5; brightness", |s| { s.push("brightness", 1_i64); }, true); + // The op-assignment expansion path: no `-=` for strings, so rhai falls + // back to `x = x - y` and fails there rather than reporting no `-=`. + agree( + "mode += \"!\"; mode", + |s| { + s.push("mode", "go".to_string()); + }, + true, + ); +} + +/// Rhai's method-call rewrite reaches a caller's variable too. +/// +/// `f(x, ..)` means `x.f(..)`, so a `&mut` first parameter mutates the +/// variable. A local is addressed by slot and one the caller supplied is not, +/// but the rule is the same for both — which is the point, since whether a +/// script's variable is a local is not something the script says. +#[test] +fn a_caller_variable_in_first_argument_position_is_taken_by_reference() { + agree("push(log, 2); log", |s| { s.push("log", vec![Dynamic::from(1_i64)]); }, true); + agree("bump(w); w.level", |s| { s.push("w", corpus::Widget::default()); }, true); + // A constant is not a place rhai will hand out, so the mutation is + // discarded — and the caller's entry has to come back untouched. + agree( + "push(log, 2); log", + |s| { s.push_constant("log", vec![Dynamic::from(1_i64)]); }, + true, + ); + // Read after the other arguments, so it is the second name reported + // missing rather than the first. + agree("nosuch(gone, missing)", |_| {}, true); + agree("nosuch(gone, brightness)", |s| { s.push("brightness", 1_i64); }, true); +} + +/// A chain rooted at a caller's variable, which is the last root shape that +/// did not lower. +/// +/// Whether such a root can be written through is not known until it is looked +/// up, and rhai decides at the same moment and the same way: `search_namespace` +/// hands back a `Target`, and a scope entry becomes a reference where anything +/// else becomes a read-only value (`eval/expr.rs:120-155`). One case per arm of +/// that, because each fails differently. +#[test] +fn a_chain_can_be_rooted_at_a_caller_variable() { + let array = || vec![Dynamic::from(1_i64)]; + + // A writable entry: read, mutate through a method, and assign through. + agree("host[0]", |s| { s.push("host", array()); }, true); + agree("host.push(2); host", |s| { s.push("host", array()); }, true); + agree("host[0] = 9; host", |s| { s.push("host", array()); }, true); + agree("host.level", |s| { s.push("host", corpus::Widget::default()); }, true); + agree("host.level = 3; host.level", |s| { s.push("host", corpus::Widget::default()); }, true); + + // A constant is not a place. Rhai refuses the assignment outright and + // refuses a mutating method too, because it never hands out a reference to + // one and a non-pure native will not take a read-only first argument + // (`func/call.rs:405`). + agree("host[0] = 9; host", |s| { s.push_constant("host", array()); }, true); + agree("host.push(2); host", |s| { s.push_constant("host", array()); }, true); + agree("host[0]", |s| { s.push_constant("host", array()); }, true); + + // A shared entry walks through its cell's guard, so the mutation lands + // where every holder of the cell can see it. The closure is made in a + // block so the compared scope does not end up holding a pointer, which the + // two sides render differently on purpose. + agree( + "{ let keep = || host.len(); } host.push(2); host", + |s| { s.push("host", array()); }, + true, + ); + + // No entry at all, reported against the variable rather than the chain. + agree("nowhere[0]", |_| {}, true); + agree("nowhere.push(1)", |_| {}, true); +} + +/// The other two things a name can resolve to, neither of which is a place. +#[test] +fn a_chain_rooted_at_a_resolved_name_cannot_be_written_through() { + let mut engine = corpus::engine(); + engine.on_var(|name, _, _| { + Ok(match name { + "injected" => Some(Dynamic::from(vec![Dynamic::from(7_i64)])), + _ => None, + }) + }); + + let mut module = rhai::Module::new(); + module.set_var("published", vec![Dynamic::from(5_i64)]); + engine.register_global_module(module.into()); + + for source in [ + // A resolver's answer is read-only, so both the write and the mutating + // method are refused. + "injected[0]", + "injected[0] = 9; injected", + "injected.push(2); injected", + // A module's constant is not, so both go into a copy and are discarded. + // The two constants are not the same constant, which is the thing worth + // pinning here (`eval/expr.rs:151` against `:122`). + "published[0]", + "published[0] = 9", + "published.push(2)", + ] { + agree_with(&engine, source, |_| {}, true); + } +} + +/// A closure can capture a variable the caller supplied, and capturing it means +/// binding the cell rather than a copy of what is in it. +/// +/// Both halves of that were wrong. Sharing found the entry by walking +/// `Scope::iter_raw`, which runs from the top down, and used the position as if +/// it counted from the bottom — so with more than one caller entry it shared +/// the wrong variable outright. And the read that binds the capture went +/// through the flattening one, so even the right variable was captured by +/// value: a write afterwards was invisible to the closure. +#[test] +fn a_closure_can_capture_a_caller_variable() { + let seed = |scope: &mut Scope| { + // Two of them, and the interesting one is not last: the index bug is + // invisible with a single entry. + scope.push("first", vec![Dynamic::from(7_i64)]); + scope.push("second", vec![Dynamic::from(1_i64)]); + }; + + // The write happens after the closure is made, so a captured copy answers + // with the old length. + agree( + "let n = 0; { let f = || first.len(); first.push(9); n = f.call(); } n", + seed, + true, + ); + agree( + "let n = 0; { let f = || second.len(); second.push(9); n = f.call(); } n", + seed, + true, + ); + // And the capture is what shares it, which `is_shared` can see. + agree("{ let f = || first.len(); } is_shared(first)", seed, true); + agree("{ let f = || first.len(); } is_shared(second)", seed, true); +} + +/// A local of the same name hides the caller's and must not write through to +/// it — the caller's entry has to come back untouched. +#[test] +fn a_local_shadows_the_caller_without_disturbing_it() { + agree( + "let brightness = 1; brightness += 1; brightness", + |s| { s.push("brightness", 100_i64); }, + true, + ); + // And the other order: read before the local exists, so the same name is + // two different variables in one script. + agree( + "let first = brightness; let brightness = 1; [first, brightness]", + |s| { s.push("brightness", 100_i64); }, + true, + ); +} + +#[test] +fn a_caller_constant_cannot_be_assigned_to() { + let constant = |s: &mut Scope| { + s.push_constant("mode", lit(1)); + }; + + agree("mode = 2; mode", constant, true); + agree("mode += 1; mode", constant, true); + // Reading one is fine, which is the half that must keep working. + agree("mode + 1", constant, true); +} + +#[test] +fn a_name_that_is_nowhere_is_reported_the_same_way() { + agree("nope + 1", |_| {}, true); + agree("nope = 1", |_| {}, true); + agree("nope += 1", |_| {}, true); +} + +/// A bare script-function name is a function pointer with the calling +/// environment attached (`eval/expr.rs:71-99`), not a variable read, so it +/// must stay a fragment rather than becoming a name lookup that fails. +/// +/// Checked on the compiled program rather than by running it, because running +/// it currently disagrees with the walker for an unrelated reason: `execute` +/// forces `always_search_scope` whenever a program has any fragment, and that +/// flag makes rhai skip the function-pointer branch entirely +/// (`eval/expr.rs:62`). Reported separately — it predates named variables, and +/// the fix is about residuals rather than about this. +#[test] +fn a_script_function_name_is_not_a_variable() { + let engine = corpus::engine(); + let ast = engine + .compile("fn helper() { 1 } let f = helper; f.call()") + .expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert!( + program.residual_count() > 0, + "a function name must not become a name lookup", + ); + assert!( + !rhaigrain::bytecode::disassemble(program.code()) + .any(|(.., op)| matches!(op, rhaigrain::bytecode::Op::LoadNamed(..))), + "nothing in {:?} may load `helper` by name", + program, + ); +} + +/// The last of the three places rhai looks: a constant a host published on a +/// module rather than in the scope. +#[test] +fn a_global_module_constant_resolves() { + let mut engine = corpus::engine(); + let mut module = Module::new(); + module.set_var("CHANNELS", 512_i64); + engine.register_global_module(module.into()); + + let ast = engine.compile("CHANNELS / 2").expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0); + + let value = Vm::new(&engine) + .run(&program, &mut Scope::new()) + .expect("a module constant must resolve"); + assert_eq!(value.as_int().unwrap(), 256); + + // And writing to one is refused, because it is a value and not a place. + let ast = engine.compile("CHANNELS = 1").expect("must compile"); + let program = Compiler::new().compile(&ast); + let err = Vm::new(&engine) + .run(&program, &mut Scope::new()) + .expect_err("a module constant is not assignable"); + assert!( + matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), + "got {err:?}", + ); +} + +/// The first of the three, and the one a VM would most plausibly skip: a +/// resolver the host registered through `Engine::on_var` sees the name before +/// the scope does. +#[test] +fn a_variable_resolver_is_consulted_first() { + let mut engine = corpus::engine(); + engine.on_var(|name, _, _| { + Ok(match name { + "injected" => Some(Dynamic::from(99_i64)), + // Declining must fall through to the scope rather than fail. + _ => None, + }) + }); + + let compile = |engine: &Engine, source: &str| { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "{source:?} must not fragment"); + program + }; + + let mut scope = Scope::new(); + scope.push("ordinary", 5_i64); + + let program = compile(&engine, "injected + ordinary"); + let value = Vm::new(&engine) + .run(&program, &mut scope.clone()) + .expect("both must resolve"); + assert_eq!(value.as_int().unwrap(), 104); + + // A resolver hands back a value rather than a place, so it is read-only. + let program = compile(&engine, "injected = 1"); + let err = Vm::new(&engine) + .run(&program, &mut scope.clone()) + .expect_err("a resolved variable is not assignable"); + assert!( + matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), + "got {err:?}", + ); + + // And the walker agrees about all of it. + let ast = engine.compile("injected + ordinary").expect("must compile"); + let expected = engine + .eval_ast_with_scope::(&mut scope.clone(), &ast) + .expect("the walker must agree"); + assert_eq!(expected.as_int().unwrap(), 104); +} + +/// A resolver that pushes onto the scope invalidates every parse-time index, +/// and rhai stops trusting them from that point. Nothing this compiler emits +/// depends on those, but a fragment's does — so the flag still has to be set. +#[test] +fn a_resolver_that_grows_the_scope_forces_a_search() { + let mut engine = corpus::engine(); + engine.on_var(|name, _, mut context| { + if name == "grow" { + context.scope_mut().push("added", 1_i64); + return Ok(Some(Dynamic::from(1_i64))); + } + Ok(None) + }); + + let ast = engine.compile("let a = 10; grow + a").expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0); + + let mut scope = Scope::new(); + let value = Vm::new(&engine) + .run(&program, &mut scope) + .expect("must run"); + + let mut walked = Scope::new(); + let expected = engine + .eval_ast_with_scope::(&mut walked, &ast) + .expect("the walker must run it too"); + + assert_eq!(format!("{value:?}"), format!("{expected:?}")); +} + +/// The receiver a resolver answered is not the scope entry of the same name. +/// +/// This is the case the by-reference rewrite is most easily got wrong on: the +/// resolver is consulted first and hands back a *value*, so `push(shadowed, 2)` +/// mutates a temporary and the caller's entry is left alone — even though there +/// is an entry of that name sitting right there to take a reference to. +/// +/// The VM tells them apart by the value already on the stack being read-only, +/// which is how `load_named` marks a resolver's answer. The alternative would +/// be running the resolver a second time, and a host can see that. +#[test] +fn a_resolved_receiver_is_not_the_scope_entry_it_shadows() { + let mut engine = corpus::engine(); + engine.on_var(|name, _, _| { + Ok(match name { + "shadowed" => Some(Dynamic::from(vec![Dynamic::from(9_i64)])), + _ => None, + }) + }); + + let source = "push(shadowed, 2); shadowed"; + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "the call must lower"); + + let start = |scope: &mut Scope| { + scope.push("shadowed", vec![Dynamic::from(1_i64)]); + }; + + let mut walked = Scope::new(); + start(&mut walked); + let walker = engine.eval_ast_with_scope::(&mut walked, &ast); + + let mut run = Scope::new(); + start(&mut run); + let ours = Vm::new(&engine).run(&program, &mut run); + + // Read-only all the way through, so rhai refuses the call outright rather + // than mutating a copy — which is a sharper thing to agree on. + assert!( + matches!( + walker.as_ref().unwrap_err().as_ref(), + rhai::EvalAltResult::ErrorNonPureMethodCallOnConstant(..), + ), + "got {walker:?}", + ); + assert_eq!(capture(&run, ours), capture(&walked, walker)); + assert_eq!( + format!("{:?}", run.get_value::("shadowed").unwrap()), + "[1]", + "the entry the resolver shadowed must come back untouched", + ); +} + +/// The one place a compiled closure is not the walker's closure. +/// +/// Rhai's parser builds a pointer that embeds the closure's `ScriptFuncDef` — +/// the AST body — and tags it with the environment it was written in. That is +/// exactly what an artifact must not carry, so the compiler emits a +/// name-only pointer to the chunk it compiled from that same body. +/// +/// Everything the closure *does* is identical; what differs is that ours is +/// late-bound, which rhai renders. Pinned here rather than left to be +/// discovered, because it is visible to a script that prints one. +#[test] +fn a_closure_pointer_is_late_bound() { + let engine = corpus::engine(); + let source = "let n = 1; let f = |x| x + n; f"; + + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "the closure must lower"); + + let ours = Vm::new(&engine) + .run(&program, &mut Scope::new()) + .expect("must run"); + let walker = engine + .eval_ast_with_scope::(&mut Scope::new(), &ast) + .expect("must run under rhai too"); + + let (ours, walker) = (format!("{ours:?}"), format!("{walker:?}")); + assert!( + ours.starts_with("Fn(\"anon$"), + "ours is a plain named pointer: {ours}", + ); + assert!( + walker.starts_with("Fn*+(\"anon$"), + "rhai's carries a script body and an environment: {walker}", + ); + + // And the difference is only in the binding: calling either gives the + // same answer, which is what the corpus covers. + assert_ne!(ours, walker, "if these ever match, delete this test"); +} + +/// Calling a compiled function from outside, which is what a native wrapper +/// will do once compiled chunks are registered for callbacks. +#[test] +fn a_compiled_function_can_be_called_by_name() { + let engine = corpus::engine(); + let ast = engine + .compile("fn add(a, b) { a + b } fn boom() { throw 7; } 0") + .expect("must compile"); + let program = Compiler::new().compile(&ast); + + let mut vm = Vm::new(&engine); + let value = vm + .call_function( + &program, + "add", + vec![Dynamic::from(2_i64), Dynamic::from(3_i64)], + 0, + rhai::Position::NONE, + ) + .expect("must call"); + assert_eq!(value.as_int().unwrap(), 5); + + // Wrong arity is a miss, not a crash — the table is keyed on both. + let err = vm + .call_function(&program, "add", vec![Dynamic::from(1_i64)], 0, rhai::Position::NONE) + .expect_err("one argument is a different function"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorFunctionNotFound(..))); + + // And what the function raises comes back, wrapped as rhai wraps it. + let err = vm + .call_function(&program, "boom", Vec::new(), 0, rhai::Position::NONE) + .expect_err("must propagate"); + assert!( + matches!(*err, rhai::EvalAltResult::ErrorInFunctionCall(..)), + "got {err:?}", + ); + + // The operand stack is where it started, so a caller can keep using it. + let value = vm + .call_function( + &program, + "add", + vec![Dynamic::from(10_i64), Dynamic::from(1_i64)], + 0, + rhai::Position::NONE, + ) + .expect("must call again"); + assert_eq!(value.as_int().unwrap(), 11); +} + +/// The flag a host uses to decide whether a program has to be owned. +/// +/// Registering compiled functions so a native can call one back requires a +/// `'static` wrapper, which means owning the program and giving up the +/// borrowed-from-the-artifact loading. Nobody should have to read a script to +/// find out whether that is needed — and the answer must be the same for a +/// compiled program and for the same program read back, or one of the two +/// paths quietly loses its callbacks. +#[test] +fn the_compiler_says_whether_a_program_makes_function_pointers() { + let engine = corpus::engine(); + + for (source, expected) in [ + ("let a = 1; a + 2", false), + ("fn f(x) { x } f(1)", false), + ("let s = 0; for i in 0..3 { s += i; } s", false), + // Every shape that produces one. + ("let n = \"ab\" + \"s\"; Fn(n)", true), + ("let n = 1; let f = |x| x + n; f.call(2)", true), + ("fn t(x) { x } let f = Fn(\"t\"); f.call(1)", true), + ("fn a(x, y) { x } let n = \"a\"; Fn(n).curry(1)", true), + ] { + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + assert_eq!( + program.makes_fn_pointers(), + expected, + "for {source:?}", + ); + + // Read off the code, so an artifact answers the same. + if let Ok(bytes) = program.write() { + let reloaded = rhaigrain::Program::read(&bytes).expect("must load"); + assert_eq!( + reloaded.makes_fn_pointers(), + expected, + "after a round trip, for {source:?}", + ); + } + } +} + +/// The whole reason for the opcode: these now cross a wire. +#[test] +fn a_program_reading_caller_state_can_be_written() { + let engine = corpus::engine(); + let source = "let out = brightness; if mode == \"chase\" { out += 10 } out"; + + let ast = engine.compile(source).expect("must compile"); + let program = Compiler::new().compile(&ast); + let bytes = program.write().expect("must be writable"); + + let reloaded = rhaigrain::Program::read(&bytes).expect("must load"); + + let mut scope = Scope::new(); + scope.push("brightness", 5_i64); + scope.push("mode", "chase".to_string()); + let value = Vm::new(&engine) + .run(&reloaded, &mut scope) + .expect("must run"); + + assert_eq!(value.as_int().unwrap(), 15); +} From 6be7e2f36fda73def2ee08b25c71e8c65883aa2d Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 17:42:15 -0700 Subject: [PATCH 02/33] all tests pass --- Cargo.toml | 17 +- src/grain/bytecode/chain.rs | 17 +- src/grain/bytecode/chunk.rs | 1 + src/grain/bytecode/code.rs | 82 ++++++++- src/grain/bytecode/mod.rs | 3 + src/grain/bytecode/op.rs | 67 +++++-- src/grain/bytecode/positions.rs | 12 +- src/grain/bytecode/strings.rs | 26 ++- src/grain/bytecode/switch.rs | 11 +- src/grain/bytecode/verify.rs | 74 +++++++- src/grain/compile/mod.rs | 2 + src/grain/format/abi.rs | 7 +- src/grain/format/mod.rs | 6 +- src/grain/format/read.rs | 33 +++- src/grain/format/write.rs | 15 +- src/grain/mod.rs | 2 +- src/grain/pos/mod.rs | 10 +- src/grain/pos/varint.rs | 5 +- src/grain/program.rs | 44 ++--- src/grain/vm/mod.rs | 11 +- tests/grain/allocation.rs | 122 +++---------- tests/grain/callback.rs | 36 +--- tests/grain/differential.rs | 164 +++++------------- tests/grain/format.rs | 243 +++++++------------------- tests/grain/fuzz.rs | 61 ++----- tests/grain/limits.rs | 105 +++-------- tests/grain/patch_integrity.rs | 32 ---- tests/grain/scope.rs | 297 ++++++++++++++++---------------- tests/mod.rs | 18 ++ 29 files changed, 697 insertions(+), 826 deletions(-) delete mode 100644 tests/grain/patch_integrity.rs create mode 100644 tests/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 702331720..f7ae7f380 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ debugging = ["internals"] ## Features and dependencies required by `bin` tools: `decimal`, `metadata`, `serde`, `debugging` and [`rustyline`](https://crates.io/crates/rustyline). bin-features = ["decimal", "metadata", "serde", "debugging", "rustyline"] ## Enable fuzzing via the [`arbitrary`](https://crates.io/crates/arbitrary) crate. -fuzz = ["arbitrary", "rust_decimal?/rust-fuzz", "serde"] +fuzz = ["arbitrary", "rust_decimal?/rust-fuzz", "serde", "grain"] ## Enable the experimental `grain` bytecode VM grain = [] @@ -149,6 +149,21 @@ required-features = ["serde"] name = "definitions" required-features = ["metadata", "internals"] +# The grain harnesses, as one binary. `tests/mod.rs` wraps them in an inline +# `mod grain`, which is what puts their module paths inside `tests/grain/`. +[[test]] +name = "mod" +path = "tests/mod.rs" +required-features = ["grain"] + +# Its own binary, and not negotiable: it installs a counting global allocator +# and reads process-global counters as deltas around each call, so any test +# allocating on another thread corrupts every number it reports. +[[test]] +name = "grain_allocation" +path = "tests/grain/allocation.rs" +required-features = ["grain"] + [profile.release] lto = "fat" codegen-units = 1 diff --git a/src/grain/bytecode/chain.rs b/src/grain/bytecode/chain.rs index b4a237834..221e8dcb9 100644 --- a/src/grain/bytecode/chain.rs +++ b/src/grain/bytecode/chain.rs @@ -28,8 +28,11 @@ pub enum Step { /// carrying one position between it and its neighbours would name the /// wrong one. Index { + /// Where the index sits on the operand stack. operand: u16, + /// Where the index expression starts. pos: rhai::Position, + /// The `[` in front of it. bracket: rhai::Position, }, @@ -43,16 +46,19 @@ pub enum Step { getter: u32, /// `set$name`, for the write-back. setter: u32, + /// Where the property is in the source. pos: rhai::Position, }, /// `.name(args)`, with the receiver as the first argument by reference. Method { + /// The name of the method name: u32, /// How many arguments, not counting the receiver. argc: u8, /// Where the first of them sits on the operand stack. operand: u16, + /// Where the call is in the source. pos: rhai::Position, }, } @@ -103,6 +109,7 @@ pub enum Root { /// to land in the scope entry, and walking a copy of it would lose the /// mutation. Local { + /// The slot index slot: u16, /// Names it in `ErrorAssignmentToConstant`. name: u32, @@ -123,7 +130,12 @@ pub enum Root { /// chain. That costs nothing extra: chain positions already live in this /// pool rather than in the strippable table, for the reason [`Step::pos`] /// gives. - Named { name: u32, pos: rhai::Position }, + Named { + /// The name of the variable + name: u32, + /// Where the variable is in the source. + pos: rhai::Position, + }, /// A value the instruction takes off the operand stack, pushed above the /// step operands. @@ -138,8 +150,11 @@ pub enum Root { /// A whole `a.b[i].c` chain. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Chain { + /// Where the chain starts. pub root: Root, + /// The steps, in source order. pub steps: Vec, + /// What happens at the end. pub tail: Tail, /// How many operand-stack values the *steps* consume, so the VM knows /// where they start. Not the whole instruction's appetite — see diff --git a/src/grain/bytecode/chunk.rs b/src/grain/bytecode/chunk.rs index e121bf9d0..b5bf4f82a 100644 --- a/src/grain/bytecode/chunk.rs +++ b/src/grain/bytecode/chunk.rs @@ -41,6 +41,7 @@ impl Chunk { self.end } + /// The deepest the operand stack gets, as proven by the verifier. #[must_use] pub fn max_stack(&self) -> u16 { self.max_stack diff --git a/src/grain/bytecode/code.rs b/src/grain/bytecode/code.rs index 69622e938..8e814af9c 100644 --- a/src/grain/bytecode/code.rs +++ b/src/grain/bytecode/code.rs @@ -35,61 +35,117 @@ use crate::grain::bytecode::{Op, Receiver}; /// Operators get their own tag rather than an optional field, so the common /// call pays nothing for the one that carries a token. pub mod tag { + /// [`Op::Const`](super::Op::Const). pub const CONST: u8 = 0x01; + /// [`Op::Unit`](super::Op::Unit). pub const UNIT: u8 = 0x02; + /// [`Op::Bool`](super::Op::Bool) holding `false`. pub const FALSE: u8 = 0x03; + /// [`Op::Bool`](super::Op::Bool) holding `true`. pub const TRUE: u8 = 0x04; + /// [`Op::LoadLocal`](super::Op::LoadLocal). pub const LOAD_LOCAL: u8 = 0x05; + /// [`Op::StoreLocal`](super::Op::StoreLocal). pub const STORE_LOCAL: u8 = 0x06; + /// [`Op::AssignLocal`](super::Op::AssignLocal) with a plain `=`. pub const ASSIGN_LOCAL: u8 = 0x07; + /// [`Op::AssignLocal`](super::Op::AssignLocal) through an operator. pub const ASSIGN_LOCAL_OP: u8 = 0x08; + /// [`Op::DeclareLocal`](super::Op::DeclareLocal) for a `let`. pub const DECLARE_LOCAL: u8 = 0x09; + /// [`Op::DeclareLocal`](super::Op::DeclareLocal) for a `const`. pub const DECLARE_CONST: u8 = 0x0a; + /// [`Op::Pop`](super::Op::Pop). pub const POP: u8 = 0x0b; + /// [`Op::Jump`](super::Op::Jump). pub const JUMP: u8 = 0x0c; + /// [`Op::JumpIfTrue`](super::Op::JumpIfTrue). pub const JUMP_IF_TRUE: u8 = 0x0d; + /// [`Op::JumpIfFalse`](super::Op::JumpIfFalse). pub const JUMP_IF_FALSE: u8 = 0x0e; + /// [`Op::Call`](super::Op::Call) to an ordinary function. pub const CALL: u8 = 0x0f; + /// [`Op::Call`](super::Op::Call) to an operator. pub const CALL_OP: u8 = 0x10; + /// [`Op::UnwindTo`](super::Op::UnwindTo). pub const UNWIND_TO: u8 = 0x11; + /// [`Op::Tick`](super::Op::Tick). pub const TICK: u8 = 0x12; + /// [`Op::Return`](super::Op::Return). pub const RETURN: u8 = 0x13; + /// [`Op::EvalAst`](super::Op::EvalAst) that rewinds the scope. pub const EVAL_AST: u8 = 0x14; + /// [`Op::EvalAst`](super::Op::EvalAst) that keeps what it declared. pub const EVAL_AST_KEEP: u8 = 0x15; + /// [`Op::Chain`](super::Op::Chain). pub const CHAIN: u8 = 0x16; + /// [`Op::MakeArray`](super::Op::MakeArray). pub const MAKE_ARRAY: u8 = 0x17; + /// [`Op::Switch`](super::Op::Switch). pub const SWITCH: u8 = 0x18; + /// [`Op::LoadNamed`](super::Op::LoadNamed). pub const LOAD_NAMED: u8 = 0x19; + /// [`Op::AssignNamed`](super::Op::AssignNamed) with a plain `=`. pub const ASSIGN_NAMED: u8 = 0x1a; + /// [`Op::AssignNamed`](super::Op::AssignNamed) through an operator. pub const ASSIGN_NAMED_OP: u8 = 0x1b; + /// [`Op::Throw`](super::Op::Throw). pub const THROW: u8 = 0x1c; + /// [`Op::IterInit`](super::Op::IterInit). pub const ITER_INIT: u8 = 0x1d; + /// [`Op::IterNext`](super::Op::IterNext). pub const ITER_NEXT: u8 = 0x1e; + /// [`Op::IterDrop`](super::Op::IterDrop). pub const ITER_DROP: u8 = 0x1f; + /// [`Op::StoreShared`](super::Op::StoreShared). pub const STORE_SHARED: u8 = 0x20; + /// [`Op::IterNext`](super::Op::IterNext) that also pushes the count. pub const ITER_NEXT_INDEXED: u8 = 0x21; + /// [`Op::PopHandler`](super::Op::PopHandler). pub const POP_HANDLER: u8 = 0x22; + /// [`Op::PushHandler`](super::Op::PushHandler) for a bare `catch`. pub const PUSH_HANDLER: u8 = 0x23; + /// [`Op::PushHandler`](super::Op::PushHandler) binding a catch variable. pub const PUSH_HANDLER_VAR: u8 = 0x24; + /// [`Op::InterpolateStart`](super::Op::InterpolateStart). pub const INTERPOLATE_START: u8 = 0x25; + /// [`Op::InterpolateAppend`](super::Op::InterpolateAppend). pub const INTERPOLATE_APPEND: u8 = 0x26; + /// [`Op::InterpolateEnd`](super::Op::InterpolateEnd). pub const INTERPOLATE_END: u8 = 0x27; + /// [`Op::MakeFnPtr`](super::Op::MakeFnPtr). pub const MAKE_FN_PTR: u8 = 0x28; + /// [`Op::Curry`](super::Op::Curry). pub const CURRY: u8 = 0x29; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) in call position. pub const CALL_FN_PTR: u8 = 0x2a; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) in method position. pub const CALL_FN_PTR_METHOD: u8 = 0x2b; + /// [`Op::Share`](super::Op::Share). pub const SHARE: u8 = 0x2c; + /// [`Op::ShareNamed`](super::Op::ShareNamed). pub const SHARE_NAMED: u8 = 0x2d; + /// [`Op::LoadShared`](super::Op::LoadShared). pub const LOAD_SHARED: u8 = 0x2e; + /// [`Op::MakeClosure`](super::Op::MakeClosure). pub const MAKE_CLOSURE: u8 = 0x2f; + /// [`Op::IsShared`](super::Op::IsShared). pub const IS_SHARED: u8 = 0x30; + /// [`Op::Checkpoint`](super::Op::Checkpoint). pub const CHECKPOINT: u8 = 0x31; + /// [`Op::CheckSize`](super::Op::CheckSize) against the array limit. pub const CHECK_ARRAY_SIZE: u8 = 0x32; + /// [`Op::CheckSize`](super::Op::CheckSize) against the map limit. pub const CHECK_MAP_SIZE: u8 = 0x33; + /// [`Op::MakeMap`](super::Op::MakeMap). pub const MAKE_MAP: u8 = 0x34; + /// [`Op::CallRef`](super::Op::CallRef) through [`Receiver::Local`](super::Receiver::Local). pub const CALL_LOCAL_REF: u8 = 0x35; + /// [`Op::Rotate`](super::Op::Rotate). pub const ROTATE: u8 = 0x36; + /// [`Op::CallRef`](super::Op::CallRef) through [`Receiver::Named`](super::Receiver::Named). pub const CALL_NAMED_REF: u8 = 0x37; + /// [`Op::LoadSharedNamed`](super::Op::LoadSharedNamed). pub const LOAD_SHARED_NAMED: u8 = 0x38; } @@ -207,14 +263,32 @@ pub fn u32_at(code: &[u8], at: usize) -> Option { pub enum AssembleError { /// A pool grew past what a `u16` operand can name. The compiler falls back /// to a whole-program fragment rather than emitting a truncated index. - PoolTooLarge { what: &'static str, entries: usize }, + PoolTooLarge { + /// Which pool overflowed + what: &'static str, + /// How many entries it holds + entries: usize, + }, /// A jump naming an instruction that does not exist. A compiler bug. - JumpOutOfRange { at: usize, target: u32 }, + JumpOutOfRange { + /// Index of the jump instruction + at: usize, + /// The instruction index it names + target: u32, + }, /// The same, for a jump that lives in a switch table rather than in the /// code. - SwitchTargetOutOfRange { table: usize, target: u32 }, + SwitchTargetOutOfRange { + /// Index into the switch pool + table: usize, + /// The instruction index the entry names + target: u32, + }, /// A chunk longer than a `u32` of bytes. - ChunkTooLarge { bytes: usize }, + ChunkTooLarge { + /// How long the chunk is + bytes: usize, + }, } /// Pack instructions into their executable form. diff --git a/src/grain/bytecode/mod.rs b/src/grain/bytecode/mod.rs index 88b176dc9..e79833c54 100644 --- a/src/grain/bytecode/mod.rs +++ b/src/grain/bytecode/mod.rs @@ -1,3 +1,6 @@ +//! The instruction set, the pools it indexes, and the checks a chunk must pass +//! before it runs. + pub mod code; mod chain; diff --git a/src/grain/bytecode/op.rs b/src/grain/bytecode/op.rs index 83191b720..e4c03b5c9 100644 --- a/src/grain/bytecode/op.rs +++ b/src/grain/bytecode/op.rs @@ -51,10 +51,10 @@ pub enum Receiver { /// One VM instruction, as the compiler emits it and a disassembly shows it. /// /// **Not the executed form.** A program's code is a byte slice, assembled from -/// these by [`assemble`](crate::bytecode::assemble) and dispatched on directly, -/// so a loaded program can borrow its instructions from the artifact rather -/// than building sixteen bytes of enum per instruction. See -/// [`code`](crate::bytecode::code) for the encoding. +/// these by [`assemble`](crate::grain::bytecode::assemble) and dispatched on +/// directly, so a loaded program can borrow its instructions from the artifact +/// rather than building sixteen bytes of enum per instruction. See +/// [`code`](crate::grain::bytecode::code) for the encoding. /// /// A stack machine: operands are pushed and consumed on an operand stack, and /// locals live in slots addressed directly. `EvalAst` is the escape hatch that @@ -64,7 +64,7 @@ pub enum Receiver { /// /// Instructions carry no source position. Several of them can fail against a /// place in the source, and the position for that comes from the program's -/// [`Positions`](crate::bytecode::Positions) table, keyed on the instruction's +/// [`Positions`](crate::grain::bytecode::Positions) table, keyed on the instruction's /// own address. Keeping it out means the diagnostics can be stripped from an /// artifact without touching the code. /// @@ -110,6 +110,7 @@ pub enum Op { /// value the resolver produced, a module's constant, a `const` — is /// `ErrorAssignmentToConstant`, as it is in the walker. AssignNamed { + /// The name of the variable name: u32, /// Index into the op-assignment pool; absent for a plain `=`. op: Option, @@ -122,6 +123,7 @@ pub enum Op { /// looks for an op-assignment implementation that mutates in place, and /// only expands to the binary form if there is none. AssignLocal { + /// The slot index slot: u16, /// Names the variable in `ErrorAssignmentToConstant`. var_name: u32, @@ -134,7 +136,12 @@ pub enum Op { /// at the top of the scope. Carries the name because locals live in the /// caller's `Scope`, where entries are named, and carries constness /// because rhai enforces it through the value's own access mode. - DeclareLocal { name: u32, is_const: bool }, + DeclareLocal { + /// The name of the variable + name: u32, + /// Whether the variable is declared `const`. + is_const: bool, + }, /// Discard the top of the operand stack. Pop, @@ -148,13 +155,19 @@ pub enum Op { /// Pop a condition and jump to `.0` if it is true. Mirrors /// [`Op::JumpIfFalse`]; both exist so short-circuit `&&` and `||` lower /// without an extra negation. - JumpIfTrue { target: u32 }, + JumpIfTrue { + /// Where to jump to + target: u32, + }, /// Pop a condition and jump to `.0` if it is false. /// /// Its position-table entry is the condition's own position, because rhai /// rejects a non-boolean guard against the guard expression rather than the /// statement — and the differential harness compares error positions. - JumpIfFalse { target: u32 }, + JumpIfFalse { + /// Where to jump to + target: u32, + }, /// Pop `argc` arguments and call the function named by `name`, pushing the /// result. @@ -173,8 +186,11 @@ pub enum Op { /// to a function pointer rather than dispatching, and a VM that did not /// would be slower than the tree it replaced. Call { + /// The name of the function name: u32, + /// How many arguments to pop argc: u8, + /// Index into the operator pool; absent unless the call is an operator. op: Option, }, @@ -198,8 +214,11 @@ pub enum Op { /// short-circuits before the rewrite (`func/call.rs:1775`), so `a + b` /// reads `a` first and needs no reference. CallRef { + /// The name of the function name: u32, + /// How many arguments to pop, not counting the receiver. argc: u8, + /// Where the first argument is found. receiver: Receiver, }, @@ -298,7 +317,12 @@ pub enum Op { /// an error: rhai takes the first argument as the pointer and binds the /// target as `this` (`func/call.rs:816-919`), which is how a closure is /// called against a receiver. - CallFnPtr { argc: u8, method: bool }, + CallFnPtr { + /// How many arguments to pop + argc: u8, + /// Whether the call is in method position (`f.call(x)`). + method: bool, + }, /// Push an empty buffer for an interpolated string to be built in. /// @@ -361,6 +385,7 @@ pub enum Op { /// keeps those positions strippable, which matters more for a literal than /// for a chain: an array can have any number of elements. CheckSize { + /// The element's index within its literal index: u16, /// Whether the element counts towards the map limit rather than the /// array one. Rhai adds one to a different member of the triple for @@ -417,7 +442,12 @@ pub enum Op { /// rewind, so they cannot disturb the scope shape slots were resolved /// against. A whole-program fragment does not, because rhai does not /// rewind top-level statements and callers can see what they declared. - EvalAst { residual: u32, rewind_scope: bool }, + EvalAst { + /// Index into the residual pool + residual: u32, + /// Whether locals the fragment declares are discarded afterwards. + rewind_scope: bool, + }, /// Arm a handler covering the instructions up to the matching /// [`Op::PopHandler`], catching to `target`. @@ -433,7 +463,12 @@ pub enum Op { /// `catch_var` names the variable the error is bound to. Its table entry /// is that variable's position, which is what rhai reports /// `ErrorTooManyVariables` against. - PushHandler { target: u32, catch_var: Option }, + PushHandler { + /// Where to jump to when an error is caught. + target: u32, + /// The name the error is bound to; absent for a bare `catch`. + catch_var: Option, + }, /// Disarm the innermost handler. /// @@ -463,9 +498,13 @@ pub enum Op { /// Its table entry is the iterable's position — `position`, not /// `start_position` — because that is what a fallible iterator's error is /// filled in with (`eval/stmt.rs:749`). - /// `indexed` is `for (x, i) in seq`: the count is pushed under the item, - /// so the two `StoreShared`s that follow pop them in declaration order. - IterNext { exit: u32, indexed: bool }, + IterNext { + /// Where to jump to once the iterator is exhausted. + exit: u32, + /// `for (x, i) in seq`: the count is pushed under the item, so the two + /// `StoreShared`s that follow pop them in declaration order. + indexed: bool, + }, /// Discard the current iterator. /// diff --git a/src/grain/bytecode/positions.rs b/src/grain/bytecode/positions.rs index 057a552d4..56ca95693 100644 --- a/src/grain/bytecode/positions.rs +++ b/src/grain/bytecode/positions.rs @@ -17,8 +17,8 @@ use crate::Position; /// back-edge passes a position to `track_operation` on every iteration, and the /// built-in operator path builds a `NativeCallContext` around one — both on the /// hot path, both needing it only if something goes wrong. Indexing an array is -/// what makes that free. The compact delta form in [`rhaigrain_pos`] is the -/// wire form, expanded once at load. +/// what makes that free. The compact delta form in [`pos`](crate::grain::pos) +/// is the wire form, expanded once at load. /// /// [`Positions::Stripped`] is not a degraded mode to apologise for: it is what /// a device runs. Errors come back carrying an instruction address instead of a @@ -61,7 +61,8 @@ impl Positions { matches!(self, Self::Stripped) } - /// Encode as the compact table [`rhaigrain_pos::resolve`] reads. + /// Encode as the compact table [`pos::resolve`](crate::grain::pos::resolve) + /// reads. /// /// Instructions with no position are skipped, which is most of them. #[must_use] @@ -89,7 +90,7 @@ impl Positions { /// /// # Errors /// - /// Whatever [`rhaigrain_pos::check`] found, or [`TableError::PastTheEnd`] + /// Whatever [`pos::check`](crate::grain::pos::check) found, or [`TableError::PastTheEnd`] /// for an address that does not name an instruction. pub fn from_table(table: &[u8], instructions: usize) -> Result { crate::grain::pos::check(table).map_err(TableError::Malformed)?; @@ -130,8 +131,11 @@ pub enum TableError { /// The table names addresses this chunk does not have, so it belongs to a /// different program. PastTheEnd { + /// How many entries the table holds entries: usize, + /// How many of them landed on an instruction matched: usize, + /// How many instructions the chunk has instructions: usize, }, } diff --git a/src/grain/bytecode/strings.rs b/src/grain/bytecode/strings.rs index 2ca3cd1b1..169346c60 100644 --- a/src/grain/bytecode/strings.rs +++ b/src/grain/bytecode/strings.rs @@ -11,10 +11,10 @@ use std::borrow::Cow; /// name is a slice of it. Two allocations for the whole table, and neither /// scales with how many names there are: the spans, and nothing else. /// -/// The exception is a name that becomes a scope entry. `Scope` stores its own -/// names, so a `let` or a parameter needs a real string — see -/// [`Program::ident`](crate::Program::ident), which is a separate and much -/// shorter pool. +/// A name that becomes a scope entry is copied rather than borrowed, because +/// `Scope` stores its own. That is a copy into a `SmartString` and not an +/// allocation, so it costs nothing for the short names a `let` or a parameter +/// actually has. #[derive(Debug, Clone, Default)] pub struct Strings<'a> { blob: Cow<'a, [u8]>, @@ -73,11 +73,13 @@ impl<'a> Strings<'a> { }) } + /// How many names the table holds. #[must_use] pub fn len(&self) -> usize { self.starts.len().saturating_sub(1) } + /// Whether the table holds no names. #[must_use] pub fn is_empty(&self) -> bool { self.len() == 0 @@ -94,20 +96,24 @@ impl<'a> Strings<'a> { core::str::from_utf8(self.blob.get(from..to)?).ok() } + /// The concatenated names, without their spans. #[must_use] pub fn blob(&self) -> &[u8] { &self.blob } + /// The span boundaries, one longer than [`Strings::len`]. #[must_use] pub fn starts(&self) -> &[u32] { &self.starts } + /// Iterate the names in index order. pub fn iter(&self) -> impl Iterator + '_ { (0..self.len() as u32).filter_map(|index| self.get(index)) } + /// Take ownership of the blob, so the table outlives the artifact. #[must_use] pub fn into_owned(self) -> Strings<'static> { Strings { @@ -123,9 +129,17 @@ pub enum BadTable { /// The spans are empty, or do not start at zero. NoTerminator, /// A span runs backwards or past the end of the blob. - SpanOutOfRange { from: usize, to: usize }, + SpanOutOfRange { + /// Where the span starts + from: usize, + /// Where it ends + to: usize, + }, /// A name is not UTF-8. - NotUtf8 { at: usize }, + NotUtf8 { + /// Where the name starts + at: usize, + }, /// The blob is longer than the last span accounts for. TrailingBytes, } diff --git a/src/grain/bytecode/switch.rs b/src/grain/bytecode/switch.rs index c3ca5a00c..defc69c9d 100644 --- a/src/grain/bytecode/switch.rs +++ b/src/grain/bytecode/switch.rs @@ -34,8 +34,7 @@ pub struct Switch { /// /// Disjoint and in ascending order, which rhai's are not: the compiler /// splits overlapping arms apart so that the first entry containing a - /// value is the only one that can match it. See - /// [`crate::compile::cases`]. + /// value is the only one that can match it. See `compile::cases`. pub ranges: Vec, /// Where to go when nothing matched. Always present: an absent `_` arm /// compiles to a jump past the statement. @@ -45,23 +44,29 @@ pub struct Switch { /// One `value => ...` arm, keyed by rhai's hash of the value. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SwitchCase { + /// Rhai's hash of the case value. pub hash: u64, + /// Where to jump to. pub target: u32, } /// One `a..b => ...` arm. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SwitchRange { + /// The lower bound pub from: INT, + /// The upper bound pub to: INT, + /// Whether the upper bound is included. pub inclusive: bool, + /// Where to jump to. pub target: u32, } impl SwitchRange { /// Whether a subject falls in this range. /// - /// Delegates to rhai's own [`RangeCase`] rather than comparing integers, + /// Delegates to rhai's own `RangeCase` rather than comparing integers, /// because a range arm matches more than integers: `switch 5.5 { 0..10 => /// .. }` matches, and under the `decimal` feature so does a `Decimal` /// (`ast/stmt.rs:254`). Rebuilding the case is two moves and no diff --git a/src/grain/bytecode/verify.rs b/src/grain/bytecode/verify.rs index a306448a9..ed2bb5fa2 100644 --- a/src/grain/bytecode/verify.rs +++ b/src/grain/bytecode/verify.rs @@ -9,12 +9,19 @@ use crate::grain::bytecode::{Chain, Chunk, Op, Receiver, Root, Step, Switch, Tai /// stack a chain consumes, and where a switch can send control. #[derive(Debug, Clone, Copy)] pub struct Pools<'a> { + /// How many constants there are. pub consts: usize, + /// How many interned names there are. pub names: usize, + /// How many operator tokens there are. pub tokens: usize, + /// How many op-assignments there are. pub assign_ops: usize, + /// How many residual AST fragments there are. pub residuals: usize, + /// The chain pool. pub chains: &'a [Chain], + /// The switch pool. pub switches: &'a [Switch], } @@ -27,42 +34,91 @@ pub struct Pools<'a> { pub enum VerifyError { /// A tag with no instruction behind it, or one whose operands run past the /// end of the chunk. - Undecodable { at: usize }, + Undecodable { + /// Byte offset of the offending tag + at: usize, + }, /// The last instruction stops short of the end, so the trailing bytes are /// not instructions. - TrailingBytes { at: usize, len: usize }, + TrailingBytes { + /// Where the trailing bytes start + at: usize, + /// How long the code is + len: usize, + }, /// A chunk names a span the code does not have. - ChunkOutOfRange { entry: u32, end: u32, len: usize }, + ChunkOutOfRange { + /// The chunk's first byte offset + entry: u32, + /// One past its last + end: u32, + /// How long the code is + len: usize, + }, /// A jump leaves the chunk it is in — including into another chunk, which /// would run that function's instructions against this frame's locals. - JumpOutOfRange { at: usize, target: u32 }, + JumpOutOfRange { + /// Byte offset of the jump + at: usize, + /// The byte offset it names + target: u32, + }, /// A jump lands inside an instruction rather than on one. Decoding from /// there would read an operand's bytes as a tag. - JumpIntoAnInstruction { at: usize, target: u32 }, + JumpIntoAnInstruction { + /// Byte offset of the jump + at: usize, + /// The byte offset it names + target: u32, + }, /// Two paths reach the same instruction with different stack depths, so /// the depth at that point is not a static property. DepthConflict { + /// Byte offset of the instruction at: usize, + /// The depth already recorded for it expected: usize, + /// The depth the other path arrives with found: usize, }, /// An instruction pops more than is on the stack. - Underflow { at: usize, need: usize, have: usize }, + Underflow { + /// Byte offset of the instruction + at: usize, + /// How many values it pops + need: usize, + /// How many are on the stack + have: usize, + }, /// Execution can run past the last instruction. FallsOffTheEnd, /// The chunk claims less stack than it uses. - StackExceedsDeclared { needed: usize, declared: u16 }, + StackExceedsDeclared { + /// The depth actually reached + needed: usize, + /// The depth the chunk declares + declared: u16, + }, /// An iterator is dropped where none was made. The compiler pairs these /// up lexically; an artifact off a wire has to be asked. - IteratorUnderflow { at: usize }, + IteratorUnderflow { + /// Byte offset of the instruction + at: usize, + }, /// A handler is disarmed where none was armed. A stale handler is worse /// than a missing one: the next unrelated error would be caught into an /// already-exited `catch` block. - HandlerUnderflow { at: usize }, + HandlerUnderflow { + /// Byte offset of the instruction + at: usize, + }, /// An index into a pool with nothing behind it. BadIndex { + /// Byte offset of the instruction at: usize, + /// Which pool was indexed what: &'static str, + /// The index it used index: u32, }, } diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index 4204e3d2f..d85903f22 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -67,11 +67,13 @@ pub struct Compiler { } impl Compiler { + /// Create a new [`Compiler`] with default options. #[must_use] pub fn new() -> Self { Self::default() } + /// Lower an `AST` into a [`Program`]. #[must_use] pub fn compile(&self, ast: &AST) -> Program<'static> { // A bare script-function name used as a value is not a variable read diff --git a/src/grain/format/abi.rs b/src/grain/format/abi.rs index 845d68da3..f4398ff30 100644 --- a/src/grain/format/abi.rs +++ b/src/grain/format/abi.rs @@ -53,7 +53,7 @@ pub struct Abi { pub int_bytes: u8, /// `size_of::()`, or 0 under `no_float`. pub float_bytes: u8, - /// [`FLAGS`] as a bitmask, low bit first. + /// `FLAGS` as a bitmask, low bit first. pub flags: u32, } @@ -65,13 +65,18 @@ pub struct Abi { pub enum AbiMismatch { /// A width differs, which means integers or floats would decode wrong. Width { + /// Which width differs what: &'static str, + /// What the writer used artifact: u8, + /// What this build uses host: u8, }, /// A restriction differs. `artifact` is whether the writer had it on. Flag { + /// Which flag differs flag: &'static str, + /// Whether the writer had it on artifact: bool, }, } diff --git a/src/grain/format/mod.rs b/src/grain/format/mod.rs index 97444ddf1..b26138d06 100644 --- a/src/grain/format/mod.rs +++ b/src/grain/format/mod.rs @@ -1,4 +1,4 @@ -//! The on-the-wire form of a [`Program`](crate::Program). +//! The on-the-wire form of a [`Program`]. //! //! This is what the project is for. A device that loads bytes runs no parser //! and builds no tree, so neither the nodes a retained rhai `AST` costs nor the @@ -27,7 +27,7 @@ //! Everything outside the code section is LEB128, signed values zigzagged, //! because it is read once at load. The code section is not: it is the bytes //! the VM executes, copied in and sliced back out untouched, with fixed-width -//! operands so dispatch does not decode. See [`crate::bytecode::code`]. +//! operands so dispatch does not decode. See [`crate::grain::bytecode::code`]. //! //! ## What it refuses to write //! @@ -115,7 +115,7 @@ impl<'a> Program<'a> { /// /// This is the split the debug layer exists for. Ship the first half to the /// device and keep the second: errors then arrive carrying an instruction - /// address, and [`rhaigrain_pos::resolve`] turns it back into a position + /// address, and [`pos::resolve`](crate::grain::pos::resolve) turns it back into a position /// where the source still is. The table can also be sent back later with /// [`Program::attach_positions`]. /// diff --git a/src/grain/format/read.rs b/src/grain/format/read.rs index 5e5838935..5ec526a55 100644 --- a/src/grain/format/read.rs +++ b/src/grain/format/read.rs @@ -23,7 +23,12 @@ pub enum ReadError { /// Not a rhaigrain artifact at all. BadMagic, /// Written by a format this build does not know how to read. - UnsupportedVersion { found: u16, supported: u16 }, + UnsupportedVersion { + /// The version the artifact claims + found: u16, + /// The version this build reads + supported: u16, + }, /// Written against a different value representation. Loading anyway would /// decode integers or floats as the wrong type. Abi(AbiMismatch), @@ -34,17 +39,33 @@ pub enum ReadError { /// A string that is not UTF-8. BadUtf8, /// A tag this build has no meaning for. - UnknownTag { section: &'static str, tag: u8 }, + UnknownTag { + /// Which section it was read from + section: &'static str, + /// The tag itself + tag: u8, + }, /// An operator syntax rhai does not recognise. - UnknownToken { syntax: String }, + UnknownToken { + /// The syntax that was read + syntax: String, + }, /// The artifact's `switch` case hashes were computed by a differently /// seeded hasher, so none of them would ever match. - HashSeedMismatch { artifact: u64, host: u64 }, - /// Constants nested past [`MAX_CONSTANT_DEPTH`]. + HashSeedMismatch { + /// The seed the writer used + artifact: u64, + /// The seed this build uses + host: u64, + }, + /// Constants nested past `MAX_CONSTANT_DEPTH`. ConstantTooDeep, /// Bytes left over after the last section, so the file is not what it /// claims to be even though every field parsed. - TrailingBytes { count: usize }, + TrailingBytes { + /// How many bytes are left over + count: usize, + }, /// The chunk parsed but does not agree with itself. Unverifiable(VerifyError), /// The position table is malformed, or belongs to a different program. diff --git a/src/grain/format/write.rs b/src/grain/format/write.rs index c8b5aa453..06a2c1ecb 100644 --- a/src/grain/format/write.rs +++ b/src/grain/format/write.rs @@ -22,8 +22,11 @@ pub enum WriteError { /// deciding whether to ship source instead needs to know what it is /// falling back for. HasResiduals { + /// How many fragments the program still has count: usize, + /// What the first of them is construct: &'static str, + /// Where it is in the source pos: rhai::Position, }, /// Script functions are still ASTs in the program's library. Compiling them @@ -31,10 +34,18 @@ pub enum WriteError { HasScriptFunctions, /// A pooled constant carries something that has no meaning in another /// process — a host type, a function pointer, a clock reading. - UnserializableConstant { index: usize, type_name: String }, + UnserializableConstant { + /// Index into the constant pool + index: usize, + /// What the constant holds + type_name: String, + }, /// An operator token that does not survive `syntax -> token`. Storing it /// would silently change which built-in the VM reaches. - AmbiguousToken { token: String }, + AmbiguousToken { + /// The token's syntax + token: String, + }, } impl core::fmt::Display for WriteError { diff --git a/src/grain/mod.rs b/src/grain/mod.rs index aae648307..5cee4eeac 100644 --- a/src/grain/mod.rs +++ b/src/grain/mod.rs @@ -14,7 +14,7 @@ //! variable access and operator fast paths are reimplemented. //! //! A program that has been lowered all the way through can be written out with -//! [`Program::write`] and read back with [`Program::read`] — see [`format`]. +//! [`Program::write`] and read back with [`Program::read`] — see [`mod@format`]. //! That is the artifact the device loads, and the reason the tree never has to //! exist there. //! diff --git a/src/grain/pos/mod.rs b/src/grain/pos/mod.rs index 54093db3a..02455d084 100644 --- a/src/grain/pos/mod.rs +++ b/src/grain/pos/mod.rs @@ -59,9 +59,15 @@ pub enum Error { Overflow, /// Addresses are not strictly ascending, so the scan's early exit would be /// wrong and a lookup could silently return the wrong site. - OutOfOrder { at: u32 }, + OutOfOrder { + /// The address that did not ascend + at: u32, + }, /// Bytes remain after the last entry, so this is not the table it claims. - TrailingBytes { count: usize }, + TrailingBytes { + /// How many bytes are left over + count: usize, + }, } impl From for Error { diff --git a/src/grain/pos/varint.rs b/src/grain/pos/varint.rs index 2e1c323cd..16526482f 100644 --- a/src/grain/pos/varint.rs +++ b/src/grain/pos/varint.rs @@ -43,7 +43,7 @@ pub fn u64(bytes: &[u8], at: &mut usize) -> Result { /// /// # Errors /// -/// As [`u64`]. +/// As [`u64`](fn@u64). pub fn i64(bytes: &[u8], at: &mut usize) -> Result { let raw = u64(bytes, at)?; Ok(((raw >> 1) as i64) ^ -((raw & 1) as i64)) @@ -53,7 +53,8 @@ pub fn i64(bytes: &[u8], at: &mut usize) -> Result { /// /// # Errors /// -/// As [`u64`], plus [`Error::Overflow`] if the value is too wide for the field. +/// As [`u64`](fn@u64), plus [`Error::Overflow`] if the value is too wide for +/// the field. pub fn u32(bytes: &[u8], at: &mut usize) -> Result { core::convert::TryFrom::try_from(u64(bytes, at)?).map_err(|_| Error::Overflow) } diff --git a/src/grain/program.rs b/src/grain/program.rs index 3170334df..d936bbd58 100644 --- a/src/grain/program.rs +++ b/src/grain/program.rs @@ -12,28 +12,12 @@ pub(crate) type SharedModule = Shared; /// A program a native function can be handed a way back into. /// -/// [`Vm::run_with_callbacks`](crate::Vm::run_with_callbacks) registers one +/// [`Vm::run_with_callbacks`](crate::grain::Vm::run_with_callbacks) registers one /// wrapper per compiled function, and rhai requires a registered function to be /// `'static` — so the program cannot still be borrowing an artifact, and the /// wrappers have to share ownership of it rather than borrow it. pub type SharedProgram = Shared>; -/// A compiled script, ready to run against an `Engine`. -/// -/// Owns everything execution needs that is not the `Engine` itself, so the -/// original `AST` can be dropped after compiling. On a small target that is the -/// whole point: the tree is the part whose cost scales with the program. -/// -/// The lifetime is the artifact's. A program read from bytes borrows its -/// instructions from them and allocates only its pools, which are bounded by -/// the distinct constants and names a script actually mentions rather than by -/// how long it is. [`Program::into_owned`] cuts the tie when that is wanted. -/// -/// `residuals` is the exception, and the reason a `Program` is not always -/// serializable. Fragments rhai's walker still has to evaluate are held as real -/// `Expr` trees, which is precisely the allocation we are trying to remove. The -/// artifact format refuses to write a `Program` that has any, so nothing -/// reaching a device can depend on them. /// One compiled script function. /// /// Called by [`Op::Call`](crate::bytecode::Op::Call) directly, without going @@ -50,6 +34,22 @@ pub struct Function { pub chunk: Chunk, } +/// A compiled script, ready to run against an `Engine`. +/// +/// Owns everything execution needs that is not the `Engine` itself, so the +/// original `AST` can be dropped after compiling. On a small target that is the +/// whole point: the tree is the part whose cost scales with the program. +/// +/// The lifetime is the artifact's. A program read from bytes borrows its +/// instructions from them and allocates only its pools, which are bounded by +/// the distinct constants and names a script actually mentions rather than by +/// how long it is. [`Program::into_owned`] cuts the tie when that is wanted. +/// +/// `residuals` is the exception, and the reason a `Program` is not always +/// serializable. Fragments rhai's walker still has to evaluate are held as real +/// `Expr` trees, which is precisely the allocation we are trying to remove. The +/// artifact format refuses to write a `Program` that has any, so nothing +/// reaching a device can depend on them. pub struct Program<'a> { /// Every chunk's instructions, concatenated: main first, then each /// function. One buffer means one position table and one instruction @@ -283,7 +283,7 @@ impl<'a> Program<'a> { /// Give up the artifact and share the program, so a native can be handed a /// way back into it. /// - /// What [`Vm::run_with_callbacks`](crate::Vm::run_with_callbacks) takes. + /// What [`Vm::run_with_callbacks`](crate::grain::Vm::run_with_callbacks) takes. /// Worth the copy only when [`makes_fn_pointers`](Self::makes_fn_pointers) /// says a pointer can escape. #[must_use] @@ -340,11 +340,13 @@ impl<'a> Program<'a> { self.recompute_max_stack(); } + /// Every chunk's instructions, concatenated. #[must_use] pub fn code(&self) -> &[u8] { &self.code } + /// The compiled script functions. #[must_use] pub fn functions(&self) -> &[Function] { &self.functions @@ -439,7 +441,7 @@ impl<'a> Program<'a> { self.switches.get(index as usize) } - /// The dispatch tables [`Op::Switch`](crate::bytecode::Op::Switch) indexes. + /// The dispatch tables [`Op::Switch`](crate::grain::bytecode::Op::Switch) indexes. /// /// Public because a disassembly that leaves them out is misleading: a /// switch's arms are reached only from its table, so without it they read @@ -456,6 +458,7 @@ impl<'a> Program<'a> { self.positions.get(pc) } + /// The whole position table, keyed on instruction address. #[must_use] pub fn positions(&self) -> &Positions { &self.positions @@ -465,7 +468,7 @@ impl<'a> Program<'a> { /// /// This is the separation the debug layer exists for: ship the program to /// the device and keep what comes back here. Errors then arrive with no - /// position, and [`rhaigrain_pos::resolve`] turns the failing instruction + /// position, and [`pos::resolve`](crate::grain::pos::resolve) turns the failing instruction /// address back into one — see [`Program::attach_positions`] for the /// inverse. pub fn strip_positions(&mut self) -> Vec { @@ -502,6 +505,7 @@ impl<'a> Program<'a> { &self.assign_ops } + /// The top-level chunk, where execution starts. #[must_use] pub fn main(&self) -> &Chunk { &self.main diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 081528564..a2b6387eb 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -283,6 +283,7 @@ struct Handler { } impl<'e> Vm<'e> { + /// A VM that dispatches through `engine`. #[must_use] pub fn new(engine: &'e Engine) -> Self { Self { @@ -312,8 +313,8 @@ impl<'e> Vm<'e> { /// exists to avoid. It cannot be helped: the outer `Vm` is borrowed by the /// frame still running beneath this one. Rhai pays the same on its own /// callbacks — but it also skips resolution entirely for a pointer that - /// carries its body, which is why a crossing measures 0.34x. See - /// [`callback`]. + /// carries its body, which is why a crossing measures 0.34x. See the + /// `callback` module. /// /// Operation counting has the same shape and the same reason: increments /// inside the callback land on the clone and are lost when it drops, as @@ -398,6 +399,7 @@ impl<'e> Vm<'e> { result } + /// Run a program's main chunk against `scope`, yielding its value. pub fn run(&mut self, program: &Program, scope: &mut Scope) -> VmResult { self.run_with(program, scope, None) } @@ -407,7 +409,8 @@ impl<'e> Vm<'e> { /// The same run, plus one native wrapper per compiled function registered /// for its duration, so a pointer this program creates resolves when rhai /// dispatches it — `let a = [1, 2]; a.map(|x| x * 2)` is `map` calling us - /// back, and `map` looks the pointer up its own way. See [`callback`]. + /// back, and `map` looks the pointer up its own way. See the `callback` + /// module. /// /// Only worth the owned program when [`Program::makes_fn_pointers`] says a /// pointer can escape; [`run`](Self::run) is otherwise identical and copies @@ -415,7 +418,7 @@ impl<'e> Vm<'e> { /// pointer simply fails to resolve, as `ErrorFunctionNotFound`, at the /// point the native tries to call it. /// - /// Read [`callback`] before relying on it: a crossing is slower than the + /// Read the `callback` module before relying on it: a crossing is slower than the /// walker, and a *capturing* closure handed to a native that binds `this` /// arrives with its arguments rotated. pub fn run_with_callbacks(&mut self, program: &SharedProgram, scope: &mut Scope) -> VmResult { diff --git a/tests/grain/allocation.rs b/tests/grain/allocation.rs index fb87acce2..5d2089d81 100644 --- a/tests/grain/allocation.rs +++ b/tests/grain/allocation.rs @@ -26,8 +26,8 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicIsize, Ordering}; +use rhai::grain::Compiler; use rhai::Engine; -use rhaigrain::Compiler; static LIVE: AtomicIsize = AtomicIsize::new(0); static COUNT: AtomicIsize = AtomicIsize::new(0); @@ -36,14 +36,7 @@ static PEAK: AtomicIsize = AtomicIsize::new(0); /// Live allocations by size class, `SIZE_CLASSES[i-1] < size <= SIZE_CLASSES[i]`. /// The small classes are what a per-allocation header punishes, and an AST is /// mostly small classes. -static BUCKETS: [AtomicIsize; 6] = [ - AtomicIsize::new(0), - AtomicIsize::new(0), - AtomicIsize::new(0), - AtomicIsize::new(0), - AtomicIsize::new(0), - AtomicIsize::new(0), -]; +static BUCKETS: [AtomicIsize; 6] = [AtomicIsize::new(0), AtomicIsize::new(0), AtomicIsize::new(0), AtomicIsize::new(0), AtomicIsize::new(0), AtomicIsize::new(0)]; const SIZE_CLASSES: [usize; 6] = [8, 16, 32, 64, 256, usize::MAX]; fn bucket_of(size: usize) -> usize { @@ -69,8 +62,7 @@ unsafe impl GlobalAlloc for Counting { } unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { - let now = LIVE.fetch_add(new as isize - l.size() as isize, Ordering::Relaxed) + new as isize - - l.size() as isize; + let now = LIVE.fetch_add(new as isize - l.size() as isize, Ordering::Relaxed) + new as isize - l.size() as isize; PEAK.fetch_max(now, Ordering::Relaxed); BUCKETS[bucket_of(l.size())].fetch_sub(1, Ordering::Relaxed); BUCKETS[bucket_of(new)].fetch_add(1, Ordering::Relaxed); @@ -203,32 +195,17 @@ fn allocation_footprint() { let per_source_byte = ast.bytes as f64 / source_bytes as f64; println!("\nsource {source_bytes} bytes"); - println!( - "\n{:<24} {:>10} {:>10} {:>10}", - "", "bytes", "allocs", "peak" - ); - println!( - "{:<24} {:>10} {:>10} {:>10}", - "Engine::new", engine_cost.bytes, engine_cost.count, engine_cost.peak - ); - println!( - "{:<24} {:>10} {:>10} {:>10}", - "engine.compile (AST)", ast.bytes, ast.count, ast.peak - ); - println!( - "{:<24} {:>10} {:>10} {:>10}", - "Compiler::compile", program.bytes, program.count, program.peak - ); + println!("\n{:<24} {:>10} {:>10} {:>10}", "", "bytes", "allocs", "peak"); + println!("{:<24} {:>10} {:>10} {:>10}", "Engine::new", engine_cost.bytes, engine_cost.count, engine_cost.peak); + println!("{:<24} {:>10} {:>10} {:>10}", "engine.compile (AST)", ast.bytes, ast.count, ast.peak); + println!("{:<24} {:>10} {:>10} {:>10}", "Compiler::compile", program.bytes, program.count, program.peak); // Not yet comparable to traffic-light's 24.0: that is a 32-bit device // figure measured against minified source, this is a host figure against // unminified source. M1 makes them comparable by measuring follow.rhai, // minified, the way the server actually ships it. println!("\nAST bytes per source byte {per_source_byte:.1} (host, unminified)"); - println!( - "AST parser peak / retained {:.2}x (the peak is what has to fit)", - ast.peak as f64 / ast.bytes as f64 - ); + println!("AST parser peak / retained {:.2}x (the peak is what has to fit)", ast.peak as f64 / ast.bytes as f64); println!("\nAST live allocations by size class"); let labels = ["<=8", "<=16", "<=32", "<=64", "<=256", ">256"]; @@ -245,20 +222,9 @@ fn allocation_footprint() { // `Expr::Variable` payload — and this script has 457 of them. Compare // against 77968, not against this, until M6 builds the restricted set. println!("\nfollow.rhai — {} source bytes", FOLLOW.len()); - println!( - "{:<24} {:>10} {:>10} {:>10}", - "engine.compile (AST)", follow_ast.bytes, follow_ast.count, follow_ast.peak - ); - println!( - "{:<24} {:>10.1}", - "bytes per source byte", - follow_ast.bytes as f64 / FOLLOW.len() as f64 - ); - println!( - "{:<24} {:>10.2}x", - "parser peak / retained", - follow_ast.peak as f64 / follow_ast.bytes as f64 - ); + println!("{:<24} {:>10} {:>10} {:>10}", "engine.compile (AST)", follow_ast.bytes, follow_ast.count, follow_ast.peak); + println!("{:<24} {:>10.1}", "bytes per source byte", follow_ast.bytes as f64 / FOLLOW.len() as f64); + println!("{:<24} {:>10.2}x", "parser peak / retained", follow_ast.peak as f64 / follow_ast.bytes as f64); println!("\nfollow.rhai AST live allocations by size class"); for (label, n) in labels.iter().zip(follow_ast.buckets) { println!(" {label:>6} {n:>8}"); @@ -273,16 +239,9 @@ fn allocation_footprint() { // The measurement the whole project was for: the real script, the real // tree, and the artifact that replaces it. let follow_program = Compiler::new().compile(&follow_ast.value); - assert_eq!( - follow_program.residual_count(), - 0, - "follow.rhai must lower completely, or there is nothing to write", - ); - let (follow_artifact, follow_table) = follow_program - .write_stripped() - .expect("follow.rhai must be writable"); - let follow_loaded = - measure(|| rhaigrain::Program::read(&follow_artifact).expect("must load")); + assert_eq!(follow_program.residual_count(), 0, "follow.rhai must lower completely, or there is nothing to write",); + let (follow_artifact, follow_table) = follow_program.write_stripped().expect("follow.rhai must be writable"); + let follow_loaded = measure(|| rhai::grain::Program::read(&follow_artifact).expect("must load")); println!( "\nfollow.rhai: {} source bytes\n tree {:>8} bytes retained, {:>5} allocs, {:>8} peak\n \ @@ -297,20 +256,14 @@ fn allocation_footprint() { follow_loaded.bytes, follow_loaded.count, ); - println!( - " {:.1}x less retained, and no parser peak at all", - follow_ast.bytes as f64 / follow_loaded.bytes as f64, - ); + println!(" {:.1}x less retained, and no parser peak at all", follow_ast.bytes as f64 / follow_loaded.bytes as f64,); // Measured at several lengths, because the ratio at one length says almost // nothing. Repeating the same statements grows the instruction stream while // the set of distinct names stays put — which is exactly the shape the // claim is about, and the shape a real script has as it gets longer. println!("\nloading an artifact against parsing the same script"); - println!( - "{:>7} {:>9} {:>9} {:>9} {:>8} {:>7}", - "source", "artifact", "tree", "loaded", "allocs", "ratio" - ); + println!("{:>7} {:>9} {:>9} {:>9} {:>8} {:>7}", "source", "artifact", "tree", "loaded", "allocs", "ratio"); let mut ratios = Vec::new(); for repeats in [1usize, 4, 16] { @@ -318,66 +271,39 @@ fn allocation_footprint() { let tree = measure(|| engine.compile(&source).expect("must compile")); let program = Compiler::new().compile(&tree.value); - assert_eq!( - program.residual_count(), - 0, - "LOADABLE must lower completely, or this measures the walker", - ); + assert_eq!(program.residual_count(), 0, "LOADABLE must lower completely, or this measures the walker",); let (stripped, _) = program.write_stripped().expect("must be writable"); - let loaded = measure(|| rhaigrain::Program::read(&stripped).expect("must load")); + let loaded = measure(|| rhai::grain::Program::read(&stripped).expect("must load")); // Borrowed, not copied: the code section contributes nothing at all to // what a load retains, which is the whole reason for the byte encoding. let code = loaded.value.code(); assert!( - code.as_ptr() >= stripped.as_ptr() - && code.as_ptr() as usize <= stripped.as_ptr() as usize + stripped.len(), + code.as_ptr() >= stripped.as_ptr() && code.as_ptr() as usize <= stripped.as_ptr() as usize + stripped.len(), "the loaded chunk must point into the artifact, not into a copy of it", ); let ratio = tree.bytes as f64 / loaded.bytes as f64; ratios.push(ratio); - println!( - "{:>7} {:>9} {:>9} {:>9} {:>8} {:>6.1}x", - source.len(), - stripped.len(), - tree.bytes, - loaded.bytes, - loaded.count, - ratio, - ); + println!("{:>7} {:>9} {:>9} {:>9} {:>8} {:>6.1}x", source.len(), stripped.len(), tree.bytes, loaded.bytes, loaded.count, ratio,); } // The property, rather than a number that would need updating: what a tree // retains grows with the program, what a load retains does not. A ratio // that stopped climbing would mean something in the loader had started // scaling with length. - assert!( - ratios[2] > ratios[1] && ratios[1] > ratios[0], - "the saving must grow with the program, got {ratios:?}", - ); + assert!(ratios[2] > ratios[1] && ratios[1] > ratios[0], "the saving must grow with the program, got {ratios:?}",); // The instrument has to be working before any milestone can lean on it. - assert!( - ast.count > 0 && ast.bytes > 0, - "the counters saw nothing; the global allocator is not installed", - ); - assert!( - ast.peak >= ast.bytes, - "peak ({}) below retained ({}) means peak tracking is broken", - ast.peak, - ast.bytes, - ); + assert!(ast.count > 0 && ast.bytes > 0, "the counters saw nothing; the global allocator is not installed",); + assert!(ast.peak >= ast.bytes, "peak ({}) below retained ({}) means peak tracking is broken", ast.peak, ast.bytes,); // M0's compiler clones the whole statement list into one residual, so the // Program is a second copy of the tree rather than a saving. Pinning that // here means M1's first real lowering shows up as this assertion failing, // which is the intended way to notice progress. - assert!( - program.bytes > 0, - "a whole-program residual should cost about what the tree costs", - ); + assert!(program.bytes > 0, "a whole-program residual should cost about what the tree costs",); drop(program); drop(ast); diff --git a/tests/grain/callback.rs b/tests/grain/callback.rs index 4395e31a1..8b343d622 100644 --- a/tests/grain/callback.rs +++ b/tests/grain/callback.rs @@ -8,11 +8,10 @@ // Only the engine is wanted here; the corpus scripts belong to the harnesses // that run all of them. -#[allow(dead_code)] -mod corpus; +use super::corpus; +use rhai::grain::{Compiler, Vm}; use rhai::{Dynamic, Engine, Scope}; -use rhaigrain::{Compiler, Vm}; /// Run a source through the VM with the callback wrappers installed. fn run(engine: &Engine, source: &str) -> Result { @@ -27,10 +26,7 @@ fn run(engine: &Engine, source: &str) -> Result { /// The same source through the walker, which is what the answer has to be. fn walk(engine: &Engine, source: &str) -> Result { let ast = engine.compile(source).map_err(|err| format!("{err:?}"))?; - engine - .eval_ast::(&ast) - .map(|value| format!("{value:?}")) - .map_err(|err| format!("{err:?}")) + engine.eval_ast::(&ast).map(|value| format!("{value:?}")).map_err(|err| format!("{err:?}")) } fn agree(source: &str) { @@ -46,10 +42,7 @@ fn lowered(source: &str) { let engine = corpus::engine(); let ast = engine.compile(source).unwrap(); let program = Compiler::new().compile(&ast); - assert!( - program.residual_count() == 0, - "{source} still fragments, so it does not test the callback path", - ); + assert!(program.residual_count() == 0, "{source} still fragments, so it does not test the callback path",); assert!(program.makes_fn_pointers(), "{source}"); } @@ -119,16 +112,8 @@ fn stock_rhai_does_the_same_to_its_own_native_pointers() { engine.register_fn("nsub", |a: i64, b: i64| a - b); let curried = "let s = \"ns\" + \"ub\"; let f = Fn(s).curry(10);"; - assert_eq!( - walk(&engine, &format!("{curried} [1, 2, 3].map(f)")), - Ok("[-9, -8, -7]".to_string()), - "rhai puts the element before the curried value", - ); - assert_eq!( - walk(&engine, &format!("{curried} f.call(1)")), - Ok("9".to_string()), - "and the curried value first when there is no element", - ); + assert_eq!(walk(&engine, &format!("{curried} [1, 2, 3].map(f)")), Ok("[-9, -8, -7]".to_string()), "rhai puts the element before the curried value",); + assert_eq!(walk(&engine, &format!("{curried} f.call(1)")), Ok("9".to_string()), "and the curried value first when there is no element",); } #[test] @@ -206,11 +191,6 @@ fn without_the_wrappers_the_pointer_does_not_resolve() { let ast = engine.compile(source).unwrap(); let program = Compiler::new().compile(&ast); - let err = Vm::new(&engine) - .run(&program, &mut Scope::new()) - .unwrap_err(); - assert!( - format!("{err:?}").contains("ErrorFunctionNotFound"), - "{err}" - ); + let err = Vm::new(&engine).run(&program, &mut Scope::new()).unwrap_err(); + assert!(format!("{err:?}").contains("ErrorFunctionNotFound"), "{err}"); } diff --git a/tests/grain/differential.rs b/tests/grain/differential.rs index 8690802ba..ebcb46010 100644 --- a/tests/grain/differential.rs +++ b/tests/grain/differential.rs @@ -10,10 +10,10 @@ //! is observable by the caller. A VM that manages its own frames could return //! the right value and still get that wrong. -mod corpus; +use super::corpus; +use rhai::grain::{Compiler, Vm}; use rhai::{Dynamic, Engine, Scope}; -use rhaigrain::{Compiler, Vm}; /// What a run produced, in a form two runs can be compared on. /// @@ -27,54 +27,39 @@ struct Outcome { } fn snapshot_scope(scope: &Scope) -> Vec<(String, String)> { - scope - .iter_raw() - .map(|(name, _, value)| (name.to_string(), format!("{value:?}"))) - .collect() + scope.iter_raw().map(|(name, _, value)| (name.to_string(), format!("{value:?}"))).collect() } fn run_stock(engine: &Engine, source: &str) -> Outcome { let mut scope = Scope::new(); - let result = engine - .compile(source) - .map_err(|err| format!("{err:?}")) - .and_then(|ast| { - engine - .eval_ast_with_scope::(&mut scope, &ast) - .map(|value| format!("{value:?}")) - .map_err(|err| format!("{err:?}")) - }); - - Outcome { - result, - scope: snapshot_scope(&scope), - } + let result = engine.compile(source).map_err(|err| format!("{err:?}")).and_then(|ast| { + engine + .eval_ast_with_scope::(&mut scope, &ast) + .map(|value| format!("{value:?}")) + .map_err(|err| format!("{err:?}")) + }); + + Outcome { result, scope: snapshot_scope(&scope) } } fn run_vm(engine: &Engine, source: &str) -> Outcome { let mut scope = Scope::new(); - let result = engine - .compile(source) + let result = engine.compile(source).map_err(|err| format!("{err:?}")).and_then(|ast| { + let program = Compiler::new().compile(&ast); + // A program that can hand a pointer to a native has to be run the + // way such a program is meant to be run, or the comparison is + // against a configuration nobody would ship. + if program.makes_fn_pointers() { + let program = program.into_shared(); + Vm::new(engine).run_with_callbacks(&program, &mut scope) + } else { + Vm::new(engine).run(&program, &mut scope) + } + .map(|value| format!("{value:?}")) .map_err(|err| format!("{err:?}")) - .and_then(|ast| { - let program = Compiler::new().compile(&ast); - // A program that can hand a pointer to a native has to be run the - // way such a program is meant to be run, or the comparison is - // against a configuration nobody would ship. - if program.makes_fn_pointers() { - let program = program.into_shared(); - Vm::new(engine).run_with_callbacks(&program, &mut scope) - } else { - Vm::new(engine).run(&program, &mut scope) - } - .map(|value| format!("{value:?}")) - .map_err(|err| format!("{err:?}")) - }); + }); - Outcome { - result, - scope: snapshot_scope(&scope), - } + Outcome { result, scope: snapshot_scope(&scope) } } #[test] @@ -96,13 +81,7 @@ fn vm_agrees_with_rhai() { } } - assert!( - failures.is_empty(), - "{} of {} corpus scripts diverged:{}", - failures.len(), - corpus::CASES.len(), - failures.join(""), - ); + assert!(failures.is_empty(), "{} of {} corpus scripts diverged:{}", failures.len(), corpus::CASES.len(), failures.join(""),); } /// The corpus is only worth anything if the comparison can actually fail. @@ -113,26 +92,10 @@ fn vm_agrees_with_rhai() { fn harness_detects_a_real_difference() { let engine = corpus::engine(); - assert_ne!( - run_stock(&engine, "1 + 1"), - run_stock(&engine, "1 + 2"), - "differing results must compare unequal", - ); - assert_ne!( - run_stock(&engine, "1"), - run_stock(&engine, "1.0"), - "int and float must not compare equal", - ); - assert_ne!( - run_stock(&engine, "let a = 1; a"), - run_stock(&engine, "1"), - "differing leftover scope must compare unequal", - ); - assert_ne!( - run_stock(&engine, "let a = [1]; a[9]"), - run_stock(&engine, "let a = [1]; a[9]"), - "the same error at a different position must compare unequal", - ); + assert_ne!(run_stock(&engine, "1 + 1"), run_stock(&engine, "1 + 2"), "differing results must compare unequal",); + assert_ne!(run_stock(&engine, "1"), run_stock(&engine, "1.0"), "int and float must not compare equal",); + assert_ne!(run_stock(&engine, "let a = 1; a"), run_stock(&engine, "1"), "differing leftover scope must compare unequal",); + assert_ne!(run_stock(&engine, "let a = [1]; a[9]"), run_stock(&engine, "let a = [1]; a[9]"), "the same error at a different position must compare unequal",); } /// A script that does not parse compares equal on both sides for the wrong @@ -144,20 +107,10 @@ fn every_corpus_script_parses() { let broken: Vec<_> = corpus::CASES .iter() - .filter_map(|case| { - engine - .compile(case.source) - .err() - .map(|err| format!("\n {}: {err}", case.name)) - }) + .filter_map(|case| engine.compile(case.source).err().map(|err| format!("\n {}: {err}", case.name))) .collect(); - assert!( - broken.is_empty(), - "{} corpus scripts do not parse, so they assert nothing:{}", - broken.len(), - broken.join(""), - ); + assert!(broken.is_empty(), "{} corpus scripts do not parse, so they assert nothing:{}", broken.len(), broken.join(""),); } /// Whether a corpus case exercises anything on this build. @@ -196,12 +149,7 @@ fn only_error_cases_error() { }) .collect(); - assert!( - surprises.is_empty(), - "{} cases fail without meaning to, so they exercise nothing:{}", - surprises.len(), - surprises.join(""), - ); + assert!(surprises.is_empty(), "{} cases fail without meaning to, so they exercise nothing:{}", surprises.len(), surprises.join(""),); } /// Corpus scripts allowed to leave a fragment behind. @@ -229,20 +177,11 @@ fn every_compiled_chunk_verifies() { .iter() .filter_map(|case| { let ast = engine.compile(case.source).ok()?; - Compiler::new() - .compile(&ast) - .verify() - .err() - .map(|err| format!("\n {}: {err:?}", case.name)) + Compiler::new().compile(&ast).verify().err().map(|err| format!("\n {}: {err:?}", case.name)) }) .collect(); - assert!( - broken.is_empty(), - "{} chunks failed verification:{}", - broken.len(), - broken.join(""), - ); + assert!(broken.is_empty(), "{} chunks failed verification:{}", broken.len(), broken.join(""),); } /// A chunk must declare the stack it uses, not the stack it might use. @@ -268,29 +207,19 @@ fn every_compiled_chunk_declares_the_stack_it_uses() { continue; }; - let declared: Vec = std::iter::once(program.main().max_stack()) - .chain(program.functions().iter().map(|f| f.chunk.max_stack())) - .collect(); + let declared: Vec = std::iter::once(program.main().max_stack()).chain(program.functions().iter().map(|f| f.chunk.max_stack())).collect(); total_declared += high_water.iter().map(|n| *n as usize).sum::(); total_ops += program.code().len(); if declared != high_water { - loose.push(format!( - "\n {}: declares {declared:?}, uses {high_water:?}", - case.name, - )); + loose.push(format!("\n {}: declares {declared:?}, uses {high_water:?}", case.name,)); } } println!("\n{total_declared} stack slots declared across {total_ops} bytes of code"); - assert!( - loose.is_empty(), - "{} chunks declare a stack they do not use:{}", - loose.len(), - loose.join(""), - ); + assert!(loose.is_empty(), "{} chunks declare a stack they do not use:{}", loose.len(), loose.join(""),); } /// Residuals are the work left to do, so the count is the progress metric. @@ -308,10 +237,7 @@ fn residual_census() { // Cases the build removed are counted on neither side, or the completeness // check below would read their absence as a corpus that stopped compiling. - let applicable = corpus::CASES - .iter() - .filter(|case| applies_to_this_build(case.name)) - .count(); + let applicable = corpus::CASES.iter().filter(|case| applies_to_this_build(case.name)).count(); for case in corpus::CASES.iter().filter(|c| applies_to_this_build(c.name)) { let Ok(ast) = engine.compile(case.source) else { @@ -332,11 +258,7 @@ fn residual_census() { } } - println!( - "\n{} of {} scripts fully lowered, {total_nodes} AST nodes still in fragments", - at_zero.len(), - corpus::CASES.len(), - ); + println!("\n{} of {} scripts fully lowered, {total_nodes} AST nodes still in fragments", at_zero.len(), corpus::CASES.len(),); println!("\nfully lowered: {}", at_zero.join(", ")); println!("\nremaining:"); remaining.sort_by_key(|(_, count)| std::cmp::Reverse(*count)); @@ -355,9 +277,5 @@ fn residual_census() { // The other direction, which the check above cannot see: a corpus that // stopped compiling at all would have nothing to fragment and would pass. - assert_eq!( - at_zero.len(), - applicable - MAY_FRAGMENT.len(), - "some scripts did not compile, so they were counted as neither", - ); + assert_eq!(at_zero.len(), applicable - MAY_FRAGMENT.len(), "some scripts did not compile, so they were counted as neither",); } diff --git a/tests/grain/format.rs b/tests/grain/format.rs index 47015ec51..30656bdea 100644 --- a/tests/grain/format.rs +++ b/tests/grain/format.rs @@ -8,11 +8,11 @@ //! every single-byte corruption of a valid artifact either loads or fails, and //! never panics. -mod corpus; +use super::corpus; +use rhai::grain::format::{ReadError, WriteError}; +use rhai::grain::{Compiler, Program, Vm}; use rhai::{Dynamic, Engine, Scope}; -use rhaigrain::format::{ReadError, WriteError}; -use rhaigrain::{Compiler, Program, Vm}; /// What a run produced, in a form two runs can be compared on. #[derive(Debug, PartialEq, Eq)] @@ -24,13 +24,8 @@ struct Outcome { /// A finished run, reduced to what two of them can be compared on. fn snapshot(scope: &Scope, result: Result>) -> Outcome { Outcome { - result: result - .map(|value| format!("{value:?}")) - .map_err(|err| format!("{err:?}")), - scope: scope - .iter_raw() - .map(|(name, _, value)| (name.to_string(), format!("{value:?}"))) - .collect(), + result: result.map(|value| format!("{value:?}")).map_err(|err| format!("{err:?}")), + scope: scope.iter_raw().map(|(name, _, value)| (name.to_string(), format!("{value:?}"))).collect(), } } @@ -89,18 +84,11 @@ fn an_artifact_runs_as_the_program_it_came_from() { let actual = run(&engine, reloaded); if expected != actual { - failures.push(format!( - "\n {name}: {source}\n rhai: {expected:?}\n artifact: {actual:?}" - )); + failures.push(format!("\n {name}: {source}\n rhai: {expected:?}\n artifact: {actual:?}")); } } - assert!( - failures.is_empty(), - "{} artifacts do not mean what they came from:{}", - failures.len(), - failures.join(""), - ); + assert!(failures.is_empty(), "{} artifacts do not mean what they came from:{}", failures.len(), failures.join(""),); } /// A round trip over an empty set passes trivially, so pin the size of the set @@ -111,39 +99,32 @@ fn the_round_trip_covers_something_worth_covering() { let written = writable(&engine); let names: Vec<_> = written.iter().map(|(name, ..)| *name).collect(); - assert!( - written.len() >= 20, - "only {} corpus scripts are writable, which is too few to prove anything: {names:?}", - written.len(), - ); + assert!(written.len() >= 20, "only {} corpus scripts are writable, which is too few to prove anything: {names:?}", written.len(),); // One per construct the encoder has a branch for, so a branch that stops // working names itself. for required in [ - "int_arithmetic", // Call with an operator token - "float_arithmetic", // a float constant, whose width the ABI pins - "shadowing_nested", // DeclareLocal and UnwindTo - "while_loop", // jumps, Tick, AssignLocal with an op - "loop_break_value", // backpatched jumps - "error_divide_by_zero", // a position that has to survive - "switch_range", // a switch table, and the hasher probe with it - "switch_guard", // and one whose arms are a chain rather than a target - "string_slice_read", // a range constant, which is a host type in `Dynamic` + "int_arithmetic", // Call with an operator token + "float_arithmetic", // a float constant, whose width the ABI pins + "shadowing_nested", // DeclareLocal and UnwindTo + "while_loop", // jumps, Tick, AssignLocal with an op + "loop_break_value", // backpatched jumps + "error_divide_by_zero", // a position that has to survive + "switch_range", // a switch table, and the hasher probe with it + "switch_guard", // and one whose arms are a chain rather than a target + "string_slice_read", // a range constant, which is a host type in `Dynamic` "string_slice_inclusive", // and the other range tag "index_assign_array", // a chain rooted at a slot, and its name "temp_root_array_method", // and one rooted on the operand stack instead ] { - assert!( - names.contains(&required), - "`{required}` no longer writes, so the encoder branch it covers is untested", - ); + assert!(names.contains(&required), "`{required}` no longer writes, so the encoder branch it covers is untested",); } } /// Where the golden pair lives. The source is checked in beside the artifact so /// a regeneration is a visible two-file change. -const GOLDEN_SOURCE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/golden.rhai"); -const GOLDEN_ARTIFACT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/golden.rgrn"); +const GOLDEN_SOURCE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/grain/fixtures/golden.rhai"); +const GOLDEN_ARTIFACT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/grain/fixtures/golden.rgrn"); /// The caller state `golden.rhai` expects. Part of the fixture, so it lives /// with it rather than being invented at each use. @@ -169,7 +150,7 @@ fn golden_scope() -> Scope<'static> { /// the question it asks is whether that was deliberate. If it was, regenerate: /// /// ```text -/// REGENERATE_GOLDEN=1 cargo test --test format golden +/// REGENERATE_GOLDEN=1 cargo test --features grain --test mod golden /// ``` /// /// and bump `VERSION` if an older reader would *misread* the new bytes rather @@ -180,12 +161,7 @@ fn a_golden_artifact_written_by_an_older_build_still_runs() { let source = std::fs::read_to_string(GOLDEN_SOURCE).expect("the golden source is checked in"); let ast = engine.compile(&source).expect("the golden source must parse"); let program = Compiler::new().compile(&ast); - assert_eq!( - program.residual_count(), - 0, - "the golden source must lower whole, or the artifact covers less than it claims: {:?}", - program.first_unsupported(), - ); + assert_eq!(program.residual_count(), 0, "the golden source must lower whole, or the artifact covers less than it claims: {:?}", program.first_unsupported(),); if std::env::var_os("REGENERATE_GOLDEN").is_some() { let bytes = program.write().expect("the golden source must be writable"); @@ -209,7 +185,7 @@ fn a_golden_artifact_written_by_an_older_build_still_runs() { Err(err) => panic!( "the golden artifact no longer loads: {err}\n\ The format moved. If that was deliberate, regenerate the fixture with \ - `REGENERATE_GOLDEN=1 cargo test --test format golden`.", + `REGENERATE_GOLDEN=1 cargo test --features grain --test mod golden`.", ), }; @@ -217,16 +193,9 @@ fn a_golden_artifact_written_by_an_older_build_still_runs() { // source is easy and silent. These are read off the *artifact*, so they say // what the encoder branch coverage actually is rather than what the source // looks like it should give. - let kinds: std::collections::BTreeSet = - rhaigrain::bytecode::disassemble(loaded.code()) - .map(|(_, op)| { - format!("{op:?}") - .split(['(', ' ', '{']) - .next() - .unwrap_or_default() - .to_string() - }) - .collect(); + let kinds: std::collections::BTreeSet = rhai::grain::bytecode::disassemble(loaded.code()) + .map(|(_, op)| format!("{op:?}").split(['(', ' ', '{']).next().unwrap_or_default().to_string()) + .collect(); for required in [ "Chain", // a chain record, with all three of its roots "CallRef", // and both by-reference call forms @@ -279,12 +248,9 @@ fn a_golden_artifact_written_by_an_older_build_still_runs() { "the golden artifact no longer means what its source means.\n\ The format moved without the reader noticing, which is the failure this \ fixture exists to catch. If the change was deliberate, regenerate with \ - `REGENERATE_GOLDEN=1 cargo test --test format golden`.", - ); - assert!( - ran.result.is_ok(), - "the golden must produce a value, not an error: {ran:?}", + `REGENERATE_GOLDEN=1 cargo test --features grain --test mod golden`.", ); + assert!(ran.result.is_ok(), "the golden must produce a value, not an error: {ran:?}",); } /// A chain rooted at a caller's variable, which the corpus cannot cover. @@ -347,10 +313,7 @@ fn refusing_to_write_names_the_construct_responsible() { let ast = engine.compile(source).expect("must compile"); let program = Compiler::new().compile(&ast); - assert!( - program.residual_count() > 0, - "{source:?} must still fragment, or this test has gone stale", - ); + assert!(program.residual_count() > 0, "{source:?} must still fragment, or this test has gone stale",); let Err(err @ WriteError::HasResiduals { construct, pos, .. }) = program.write() else { panic!("{source:?} must refuse to write"); @@ -376,19 +339,12 @@ fn script_functions_survive_the_round_trip() { ] { let ast = engine.compile(source).expect("must compile"); let program = Compiler::new().compile(&ast); - assert!( - !program.functions().is_empty(), - "{source:?} must compile its functions, not leave them to the walker", - ); + assert!(!program.functions().is_empty(), "{source:?} must compile its functions, not leave them to the walker",); let bytes = program.write().expect("must be writable"); let reloaded = Program::read(&bytes).expect("must load"); - assert_eq!( - run(&engine, reloaded), - run_stock(&engine, source), - "{source:?} does not mean the same after a round trip", - ); + assert_eq!(run(&engine, reloaded), run_stock(&engine, source), "{source:?} does not mean the same after a round trip",); } } @@ -399,23 +355,11 @@ fn script_functions_survive_the_round_trip() { fn a_function_the_compiler_cannot_lower_refuses_to_write() { let engine = corpus::engine(); // `this` is not a scope entry, so no slot addresses it. - let ast = engine - .compile("fn double() { this * 2 } let x = 21; x.double()") - .expect("must compile"); + let ast = engine.compile("fn double() { this * 2 } let x = 21; x.double()").expect("must compile"); let program = Compiler::new().compile(&ast); - assert!( - program.functions().is_empty(), - "a body using `this` must not become a chunk", - ); - assert!( - matches!( - program.write(), - Err(WriteError::HasScriptFunctions | WriteError::HasResiduals { .. }), - ), - "got {:?}", - program.write(), - ); + assert!(program.functions().is_empty(), "a body using `this` must not become a chunk",); + assert!(matches!(program.write(), Err(WriteError::HasScriptFunctions | WriteError::HasResiduals { .. }),), "got {:?}", program.write(),); } /// A program with one of everything the corruption tests need to reach: a @@ -435,19 +379,13 @@ fn sample(engine: &Engine) -> Vec { switch a { 1 => \"one\", 0..=20 => \"some\", _ => \"many\" }", ) .expect("must compile"); - Compiler::new() - .compile(&ast) - .write() - .expect("the sample must be writable") + Compiler::new().compile(&ast).write().expect("the sample must be writable") } #[test] fn something_that_is_not_an_artifact_is_refused_at_the_first_bytes() { assert_eq!(Program::read(b"").unwrap_err(), ReadError::Truncated); - assert_eq!( - Program::read(b"not an artifact at all").unwrap_err(), - ReadError::BadMagic, - ); + assert_eq!(Program::read(b"not an artifact at all").unwrap_err(), ReadError::BadMagic,); } #[test] @@ -457,10 +395,7 @@ fn a_future_format_version_is_refused_rather_than_guessed_at() { bytes[4] = 0xff; bytes[5] = 0xff; - assert!(matches!( - Program::read(&bytes).unwrap_err(), - ReadError::UnsupportedVersion { found: 0xffff, .. }, - )); + assert!(matches!(Program::read(&bytes).unwrap_err(), ReadError::UnsupportedVersion { found: 0xffff, .. },)); } /// The fingerprint is the difference between a clean failure and integers @@ -472,18 +407,12 @@ fn a_different_value_representation_is_refused_by_name() { let mut narrow = sample(&engine); narrow[6] = 4; // INT width let message = Program::read(&narrow).unwrap_err().to_string(); - assert!( - message.contains("INT") && message.contains('4'), - "the message must name the width: {message}", - ); + assert!(message.contains("INT") && message.contains('4'), "the message must name the width: {message}",); let mut restricted = sample(&engine); restricted[8] ^= 0b100; // the `no_index` bit let message = Program::read(&restricted).unwrap_err().to_string(); - assert!( - message.contains("no_index"), - "the message must name the flag: {message}", - ); + assert!(message.contains("no_index"), "the message must name the flag: {message}",); } /// A `switch` carries hashes rhai's parser computed, and rhai seeds its hasher @@ -500,24 +429,15 @@ fn a_switch_hashed_by_a_different_seed_is_refused() { // The probe is the only place the artifact repeats this value, and finding // it that way means the test does not have to know the layout. - let probe = rhaigrain::bytecode::probe().to_le_bytes(); - let at = bytes - .windows(probe.len()) - .position(|window| window == probe) - .expect("an artifact with a switch in it carries a probe"); + let probe = rhai::grain::bytecode::probe().to_le_bytes(); + let at = bytes.windows(probe.len()).position(|window| window == probe).expect("an artifact with a switch in it carries a probe"); let mut corrupt = bytes.clone(); corrupt[at] ^= 1; let err = Program::read(&corrupt).expect_err("a foreign hasher must be refused"); - assert!( - matches!(err, ReadError::HashSeedMismatch { .. }), - "got {err:?}", - ); - assert!( - err.to_string().contains("set_hashing_seed"), - "the message must say how to fix it: {err}", - ); + assert!(matches!(err, ReadError::HashSeedMismatch { .. }), "got {err:?}",); + assert!(err.to_string().contains("set_hashing_seed"), "the message must say how to fix it: {err}",); // And the uncorrupted one still loads, so the check is not simply always // failing. @@ -532,11 +452,7 @@ fn every_truncation_fails_cleanly() { let bytes = sample(&engine); for cut in 0..bytes.len() { - assert!( - Program::read(&bytes[..cut]).is_err(), - "a {cut}-byte prefix of a {}-byte artifact loaded", - bytes.len(), - ); + assert!(Program::read(&bytes[..cut]).is_err(), "a {cut}-byte prefix of a {}-byte artifact loaded", bytes.len(),); } assert!(Program::read(&bytes).is_ok(), "the whole thing must load"); @@ -550,10 +466,7 @@ fn trailing_bytes_are_refused() { let mut bytes = sample(&engine); bytes.push(0); - assert_eq!( - Program::read(&bytes).unwrap_err(), - ReadError::TrailingBytes { count: 1 }, - ); + assert_eq!(Program::read(&bytes).unwrap_err(), ReadError::TrailingBytes { count: 1 },); } /// The safety claim in one test: a corrupted artifact is a `Result`, never a @@ -587,9 +500,7 @@ fn no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk() { if let Ok(program) = Program::read(&corrupt) { loaded += 1; - program - .verify() - .expect("read must not return a chunk that fails verification"); + program.verify().expect("read must not return a chunk that fails verification"); // The result is free to be anything; not crashing is the claim. let _ = Vm::new(&engine).run(&program, &mut Scope::new()); } @@ -599,10 +510,7 @@ fn no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk() { // Most flips land in a length, a tag or the fingerprint and are rejected. // Some land in a constant's value and legitimately still load; that is the // case worth having run above. - println!( - "{loaded} of {} single-bit corruptions still loaded", - bytes.len() * 8, - ); + println!("{loaded} of {} single-bit corruptions still loaded", bytes.len() * 8,); } /// The whole point of the split, end to end. @@ -623,41 +531,25 @@ fn a_stripped_program_reports_an_address_the_host_can_resolve() { // Device: run bytes, with no table and no source. let device = Program::read(&shipped).expect("the device must load it"); - assert!( - device.positions().is_stripped(), - "a stripped artifact must not carry positions", - ); + assert!(device.positions().is_stripped(), "a stripped artifact must not carry positions",); let mut vm = Vm::new(&engine); - let error = vm - .run(&device, &mut Scope::new()) - .expect_err("dividing by zero must fail"); + let error = vm.run(&device, &mut Scope::new()).expect_err("dividing by zero must fail"); let address = vm.fault_pc().expect("a failed run must name an instruction"); // Host: resolve what came back. - let site = rhaigrain_pos::resolve(&table, address as u32) - .expect("the failing instruction must have a recorded site"); + let site = rhai::grain::pos::resolve(&table, address as u32).expect("the failing instruction must have a recorded site"); - assert_eq!( - (site.line, site.column), - (3, 3), - "the division is at line 3, column 3 of {source:?}", - ); + assert_eq!((site.line, site.column), (3, 3), "the division is at line 3, column 3 of {source:?}",); // And the same program with its table attached says so itself, exactly as // rhai does — which is what makes the resolved site trustworthy. let mut reattached = Program::read(&shipped).unwrap(); - reattached - .attach_positions(&table) - .expect("its own table must attach"); + reattached.attach_positions(&table).expect("its own table must attach"); assert_eq!(run(&engine, reattached), expected); // The stripped run is the same failure, minus the position. - assert!( - error.position().is_none(), - "a stripped program has no position to report, got {:?}", - error.position(), - ); + assert!(error.position().is_none(), "a stripped program has no position to report, got {:?}", error.position(),); } /// Attaching another program's table would misreport every error rather than @@ -667,17 +559,13 @@ fn a_table_from_a_different_program_is_refused() { let engine = corpus::engine(); let short = Compiler::new().compile(&engine.compile("1 + 1").unwrap()); - let long = Compiler::new() - .compile(&engine.compile("let a = 1; while a < 9 { a += 1 } a").unwrap()); + let long = Compiler::new().compile(&engine.compile("let a = 1; while a < 9 { a += 1 } a").unwrap()); let (_, long_table) = long.write_stripped().expect("must be writable"); let (short_bytes, _) = short.write_stripped().expect("must be writable"); let mut program = Program::read(&short_bytes).unwrap(); - assert!( - program.attach_positions(&long_table).is_err(), - "a table naming instructions this chunk does not have must be refused", - ); + assert!(program.attach_positions(&long_table).is_err(), "a table naming instructions this chunk does not have must be refused",); } /// A stripped artifact that arrives with a table still in it is a contradiction @@ -688,10 +576,7 @@ fn an_artifact_carrying_a_mismatched_table_does_not_load() { let bytes = sample(&engine); let program = Program::read(&bytes).expect("the sample must load"); - assert!( - !program.positions().is_stripped(), - "`write` keeps the table, so this one must have positions", - ); + assert!(!program.positions().is_stripped(), "`write` keeps the table, so this one must have positions",); } /// What the split costs, and what it saves. @@ -719,10 +604,7 @@ fn stripping_positions_shrinks_the_artifact() { 100.0 * (with - without) as f64 / with as f64, ); - assert!( - without < with, - "stripping must actually remove something: {without} vs {with}", - ); + assert!(without < with, "stripping must actually remove something: {without} vs {with}",); } /// The number this project exists to move: bytes retained per source byte, @@ -752,17 +634,10 @@ fn artifact_size_census() { for (name, source, artifact) in &rows { println!("{source:>7} {artifact:>7} {name}"); } - println!( - "\n{} scripts: {source_bytes} source bytes -> {artifact_bytes} artifact bytes ({:.2}x)", - rows.len(), - artifact_bytes as f64 / source_bytes as f64, - ); + println!("\n{} scripts: {source_bytes} source bytes -> {artifact_bytes} artifact bytes ({:.2}x)", rows.len(), artifact_bytes as f64 / source_bytes as f64,); // Not a target, a tripwire. The plan is explicit that bytecode need not // beat minified source on bytes — but an encoding several times larger // than its input has a bug in it, not a tradeoff. - assert!( - artifact_bytes < source_bytes * 3, - "{artifact_bytes} artifact bytes for {source_bytes} of source is not an encoding", - ); + assert!(artifact_bytes < source_bytes * 3, "{artifact_bytes} artifact bytes for {source_bytes} of source is not an encoding",); } diff --git a/tests/grain/fuzz.rs b/tests/grain/fuzz.rs index 4ffe50a8a..bcc59a684 100644 --- a/tests/grain/fuzz.rs +++ b/tests/grain/fuzz.rs @@ -17,13 +17,12 @@ // Only the sources are wanted here; the names belong to the harnesses that // report per-case results. -#[allow(dead_code)] -mod corpus; +use super::corpus; +use rhai::grain::{Compiler, Program, Vm}; use rhai::{Dynamic, Engine, Scope}; -use rhaigrain::{Compiler, Program, Vm}; -use corpus::generate::{Generator, Rng}; +use super::corpus::generate::{Generator, Rng}; /// Ways an artifact can arrive wrong. Truncation and splicing matter as much /// as corruption: a length field that disagrees with what follows is how a @@ -92,11 +91,7 @@ fn mutated_artifacts_load_or_fail_but_never_misbehave() { }) .collect(); - assert!( - seeds.len() >= 50, - "only {} artifacts to mutate, which is too few to prove anything", - seeds.len(), - ); + assert!(seeds.len() >= 50, "only {} artifacts to mutate, which is too few to prove anything", seeds.len(),); // A budget, because verification proves structure and not termination: a // corrupted-but-in-range jump target is a valid infinite loop. @@ -137,14 +132,8 @@ fn mutated_artifacts_load_or_fail_but_never_misbehave() { } } - println!( - "{loaded} of {} mutations loaded and ran", - ROUNDS * seeds.len(), - ); - assert!( - loaded > 0, - "no mutation survived, so nothing was actually executed", - ); + println!("{loaded} of {} mutations loaded and ran", ROUNDS * seeds.len(),); + assert!(loaded > 0, "no mutation survived, so nothing was actually executed",); } /// Bytes that were never an artifact, which is the other way in. @@ -190,13 +179,8 @@ impl std::fmt::Debug for Outcome { fn snapshot(scope: &Scope, result: Result>) -> Outcome { Outcome { - result: result - .map(|value| format!("{value:?}")) - .map_err(|err| format!("{err:?}")), - scope: scope - .iter_raw() - .map(|(name, _, value)| (name.to_string(), format!("{value:?}"))) - .collect(), + result: result.map(|value| format!("{value:?}")).map_err(|err| format!("{err:?}")), + scope: scope.iter_raw().map(|(name, _, value)| (name.to_string(), format!("{value:?}"))).collect(), } } @@ -212,9 +196,7 @@ fn hit_a_limit(outcome: &Outcome) -> bool { let Err(err) = &outcome.result else { return false; }; - ["ErrorTooManyOperations", "ErrorStackOverflow", "ErrorTooManyVariables"] - .iter() - .any(|limit| err.contains(limit)) + ["ErrorTooManyOperations", "ErrorStackOverflow", "ErrorTooManyVariables"].iter().any(|limit| err.contains(limit)) } /// Run something that may panic, without the panic reaching the console. @@ -265,9 +247,7 @@ fn rhai_drops_a_local_its_optimizer_still_refers_to() { plain.set_optimization_level(rhai::OptimizationLevel::None); let ast = engine.compile(WRONG).expect("it parses"); - let value = engine - .eval_ast::(&ast) - .expect("rhai runs it, which is the problem"); + let value = engine.eval_ast::(&ast).expect("rhai runs it, which is the problem"); assert_eq!( value.as_int().ok(), Some(1), @@ -306,8 +286,7 @@ fn agree_unoptimised(source: &str) -> bool { }; let mut walker_scope = Scope::new(); - let Some(walked) = quietly(|| plain.eval_ast_with_scope::(&mut walker_scope, &ast)) - else { + let Some(walked) = quietly(|| plain.eval_ast_with_scope::(&mut walker_scope, &ast)) else { return false; }; @@ -384,8 +363,7 @@ fn generated_scripts_agree_with_the_walker() { // nothing to compare against a side that did not finish, so those are // counted and dropped rather than blamed on the VM. let mut walker_scope = Scope::new(); - let Some(walked) = quietly(|| engine.eval_ast_with_scope::(&mut walker_scope, &ast)) - else { + let Some(walked) = quietly(|| engine.eval_ast_with_scope::(&mut walker_scope, &ast)) else { walker_panics += 1; continue; }; @@ -421,18 +399,13 @@ fn generated_scripts_agree_with_the_walker() { // Not every disagreement is one to have: rhai's optimizer can delete a // `let` whose variable is still read, and then there is no agreeing // with it. See `rhai_drops_a_local_its_optimizer_still_refers_to`. - if format!("{ours:?}").contains("ErrorVariableNotFound") - && agree_unoptimised(&source) - { + if format!("{ours:?}").contains("ErrorVariableNotFound") && agree_unoptimised(&source) { lost_a_local += 1; continue; } if failures.len() < 5 { - failures.push(format!( - "\n=== script {n} (seed {:#x}) ===\n {source}\n rhai: {walked:?}\n vm: {ours:?}", - SEED ^ n as u64, - )); + failures.push(format!("\n=== script {n} (seed {:#x}) ===\n {source}\n rhai: {walked:?}\n vm: {ours:?}", SEED ^ n as u64,)); } } @@ -494,9 +467,5 @@ fn generated_scripts_agree_with_the_walker() { deciding the outcome rather than the code", ); - assert!( - failures.is_empty(), - "generated scripts diverged:{}", - failures.join(""), - ); + assert!(failures.is_empty(), "generated scripts diverged:{}", failures.join(""),); } diff --git a/tests/grain/limits.rs b/tests/grain/limits.rs index 19bab1c79..b54fbe6e5 100644 --- a/tests/grain/limits.rs +++ b/tests/grain/limits.rs @@ -14,9 +14,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use rhai::grain::bytecode::Op; +use rhai::grain::{Compiler, Program, Vm}; use rhai::{Dynamic, Engine, EvalAltResult, Scope}; -use rhaigrain::bytecode::Op; -use rhaigrain::{Compiler, Program, Vm}; /// A bare infinite loop, which the compiler lowers with nothing left over — /// asserted below, so this cannot silently become a test of the fallback. @@ -26,21 +26,11 @@ fn run_vm(engine: &Engine, source: &str) -> Result> let ast = engine.compile(source).expect("must compile"); let program = Compiler::new().compile(&ast); - assert_eq!( - program.residual_count(), - 0, - "{source:?} must be fully lowered, or this tests rhai rather than the VM", - ); + assert_eq!(program.residual_count(), 0, "{source:?} must be fully lowered, or this tests rhai rather than the VM",); // Without a tick on the back-edge nothing in a compiled loop ever reaches // `track_operation`, and the tests below would hang rather than fail. - assert!( - program - .main() - .ops(program.code()) - .any(|(_, op)| op == Op::Tick), - "{source:?} lowered to a loop with no operation tick", - ); + assert!(program.main().ops(program.code()).any(|(_, op)| op == Op::Tick), "{source:?} lowered to a loop with no operation tick",); Vm::new(engine).run(&program, &mut Scope::new()) } @@ -52,10 +42,7 @@ fn compiled_loop_hits_the_operation_limit() { let err = run_vm(&engine, SPIN).expect_err("an unbounded loop must be stopped"); - assert!( - matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), - "expected ErrorTooManyOperations, got {err:?}", - ); + assert!(matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), "expected ErrorTooManyOperations, got {err:?}",); } #[test] @@ -72,14 +59,8 @@ fn compiled_loop_honours_the_progress_interrupt() { let err = run_vm(&engine, SPIN).expect_err("the interrupt must stop the loop"); - assert!( - matches!(*err, EvalAltResult::ErrorTerminated(..)), - "expected ErrorTerminated, got {err:?}", - ); - assert!( - ticks.load(Ordering::SeqCst) >= 500, - "on_progress should have been called on every back-edge", - ); + assert!(matches!(*err, EvalAltResult::ErrorTerminated(..)), "expected ErrorTerminated, got {err:?}",); + assert!(ticks.load(Ordering::SeqCst) >= 500, "on_progress should have been called on every back-edge",); } /// A chunk that loops with no tick in it must still be stopped. @@ -106,39 +87,21 @@ fn a_loop_with_its_tick_removed_still_hits_the_limit() { // Where the tick sits inside the code, and what the code looks like, so the // same bytes can be found again inside the finished artifact. let code = program.code().to_vec(); - let (tick_at, _) = program - .main() - .ops(program.code()) - .find(|(_, op)| *op == Op::Tick) - .expect("the compiler ticks a loop"); + let (tick_at, _) = program.main().ops(program.code()).find(|(_, op)| *op == Op::Tick).expect("the compiler ticks a loop"); let mut bytes = program.write().expect("a lowered program must write"); - let start = bytes - .windows(code.len()) - .position(|window| window == code) - .expect("the artifact embeds the code verbatim"); + let start = bytes.windows(code.len()).position(|window| window == code).expect("the artifact embeds the code verbatim"); // `Checkpoint` is the other one-byte instruction that does nothing to the // stack, so this swap leaves every offset, jump target and position entry // exactly where it was. Only the metering goes. - bytes[start + tick_at] = rhaigrain::bytecode::code::tag::CHECKPOINT; + bytes[start + tick_at] = rhai::grain::bytecode::code::tag::CHECKPOINT; let tickless = Program::read(&bytes).expect("still a valid artifact"); - assert!( - !tickless - .main() - .ops(tickless.code()) - .any(|(_, op)| op == Op::Tick), - "the tick should be gone, or this tests nothing", - ); - - let err = Vm::new(&engine) - .run(&tickless, &mut Scope::new()) - .expect_err("a tickless loop must still be stopped"); - assert!( - matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), - "expected ErrorTooManyOperations, got {err:?}", - ); + assert!(!tickless.main().ops(tickless.code()).any(|(_, op)| op == Op::Tick), "the tick should be gone, or this tests nothing",); + + let err = Vm::new(&engine).run(&tickless, &mut Scope::new()).expect_err("a tickless loop must still be stopped"); + assert!(matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), "expected ErrorTooManyOperations, got {err:?}",); } /// The walker and the VM must agree that the script *fails*, even though they @@ -149,14 +112,9 @@ fn the_walker_agrees_the_loop_is_stopped() { engine.set_max_operations(10_000); let ast = engine.compile(SPIN).expect("must compile"); - let err = engine - .eval_ast_with_scope::(&mut Scope::new(), &ast) - .expect_err("rhai must stop it too"); - - assert!( - matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), - "expected ErrorTooManyOperations, got {err:?}", - ); + let err = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect_err("rhai must stop it too"); + + assert!(matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), "expected ErrorTooManyOperations, got {err:?}",); } /// `max_string_size` is a host's defence, and interpolation is the easiest way @@ -178,22 +136,11 @@ fn interpolation_respects_the_string_limit() { let program = Compiler::new().compile(&ast); assert_eq!(program.residual_count(), 0, "must be lowered, not walked"); - let walker = engine - .eval_ast_with_scope::(&mut Scope::new(), &ast) - .expect_err("the walker must refuse it"); - let vm = Vm::new(&engine) - .run(&program, &mut Scope::new()) - .expect_err("and so must the VM"); - - assert!( - matches!(*vm, EvalAltResult::ErrorDataTooLarge(..)), - "got {vm:?}", - ); - assert_eq!( - format!("{vm:?}"), - format!("{walker:?}"), - "including the position of the segment that went over", - ); + let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect_err("the walker must refuse it"); + let vm = Vm::new(&engine).run(&program, &mut Scope::new()).expect_err("and so must the VM"); + + assert!(matches!(*vm, EvalAltResult::ErrorDataTooLarge(..)), "got {vm:?}",); + assert_eq!(format!("{vm:?}"), format!("{walker:?}"), "including the position of the segment that went over",); } /// A loop that does terminate must not be killed by the tick itself, and must @@ -207,13 +154,9 @@ fn ticking_does_not_disturb_a_bounded_loop() { let ast = engine.compile(source).expect("must compile"); let program = Compiler::new().compile(&ast); - let vm = Vm::new(&engine) - .run(&program, &mut Scope::new()) - .expect("bounded loop must finish"); + let vm = Vm::new(&engine).run(&program, &mut Scope::new()).expect("bounded loop must finish"); - let walker = engine - .eval_ast_with_scope::(&mut Scope::new(), &ast) - .expect("bounded loop must finish under rhai too"); + let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect("bounded loop must finish under rhai too"); assert_eq!(format!("{vm:?}"), format!("{walker:?}")); } diff --git a/tests/grain/patch_integrity.rs b/tests/grain/patch_integrity.rs deleted file mode 100644 index dcc3bd0f4..000000000 --- a/tests/grain/patch_integrity.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! The fork must stay reviewable. -//! -//! rhaigrain builds against a vendored rhai rather than the published crate, so -//! that the VM can reach a handful of internals rhai does not export. That is -//! only defensible while the delta is small, visibility-only, and visible in -//! one file. `vendor/rhai` is generated and gitignored; `vendor/rhai-1.25.1.patch` -//! is the checked-in artifact, and this test is what keeps them honest. -//! -//! The real work is in `scripts/check-patch.sh`, which verifies two separate -//! claims: that the patch equals the true pristine -> vendored delta, and that -//! applying it to a pristine tree reproduces what we compile against. - -use std::process::Command; - -#[test] -fn vendored_rhai_matches_its_patch() { - let root = env!("CARGO_MANIFEST_DIR"); - - let output = Command::new("bash") - .arg(format!("{root}/scripts/check-patch.sh")) - .output() - .expect("failed to run scripts/check-patch.sh"); - - if !output.status.success() { - panic!( - "vendored rhai does not match its patch\n\ - --- stdout ---\n{}\n--- stderr ---\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - ); - } -} diff --git a/tests/grain/scope.rs b/tests/grain/scope.rs index 09826e30d..bdd26737e 100644 --- a/tests/grain/scope.rs +++ b/tests/grain/scope.rs @@ -17,11 +17,10 @@ // Only the engine is wanted here; the corpus scripts belong to the harnesses // that run all of them. -#[allow(dead_code)] -mod corpus; +use super::corpus; +use rhai::grain::{Compiler, Vm}; use rhai::{Dynamic, Engine, Module, Scope}; -use rhaigrain::{Compiler, Vm}; /// What a run produced, in a form two runs can be compared on. #[derive(Debug, PartialEq, Eq)] @@ -32,13 +31,8 @@ struct Outcome { fn capture(scope: &Scope, result: Result>) -> Outcome { Outcome { - result: result - .map(|value| format!("{value:?}")) - .map_err(|err| format!("{err:?}")), - scope: scope - .iter_raw() - .map(|(name, _, value)| (name.to_string(), format!("{value:?}"))) - .collect(), + result: result.map(|value| format!("{value:?}")).map_err(|err| format!("{err:?}")), + scope: scope.iter_raw().map(|(name, _, value)| (name.to_string(), format!("{value:?}"))).collect(), } } @@ -61,19 +55,11 @@ fn agree_with(engine: &Engine, source: &str, build: impl Fn(&mut Scope), writabl let ast = engine.compile(source).expect("must compile"); let program = Compiler::new().compile(&ast); - assert_eq!( - program.residual_count() == 0, - writable, - "{source:?} fragments: {:?}", - program.first_unsupported(), - ); + assert_eq!(program.residual_count() == 0, writable, "{source:?} fragments: {:?}", program.first_unsupported(),); let mut walked = Scope::new(); build(&mut walked); - let expected = capture( - &walked.clone(), - engine.eval_ast_with_scope::(&mut walked, &ast), - ); + let expected = capture(&walked.clone(), engine.eval_ast_with_scope::(&mut walked, &ast)); let expected = Outcome { scope: capture(&walked, Ok(Dynamic::UNIT)).scope, ..expected @@ -95,7 +81,13 @@ fn lit(value: i64) -> Dynamic { #[test] fn a_caller_variable_can_be_read() { - agree("brightness * 2", |s| { s.push("brightness", 21_i64); }, true); + agree( + "brightness * 2", + |s| { + s.push("brightness", 21_i64); + }, + true, + ); agree( "mode + \"!\"", |s| { @@ -107,8 +99,20 @@ fn a_caller_variable_can_be_read() { #[test] fn a_caller_variable_can_be_written() { - agree("brightness = 7; brightness", |s| { s.push("brightness", 1_i64); }, true); - agree("brightness += 5; brightness", |s| { s.push("brightness", 1_i64); }, true); + agree( + "brightness = 7; brightness", + |s| { + s.push("brightness", 1_i64); + }, + true, + ); + agree( + "brightness += 5; brightness", + |s| { + s.push("brightness", 1_i64); + }, + true, + ); // The op-assignment expansion path: no `-=` for strings, so rhai falls // back to `x = x - y` and fails there rather than reporting no `-=`. agree( @@ -128,19 +132,39 @@ fn a_caller_variable_can_be_written() { /// script's variable is a local is not something the script says. #[test] fn a_caller_variable_in_first_argument_position_is_taken_by_reference() { - agree("push(log, 2); log", |s| { s.push("log", vec![Dynamic::from(1_i64)]); }, true); - agree("bump(w); w.level", |s| { s.push("w", corpus::Widget::default()); }, true); + agree( + "push(log, 2); log", + |s| { + s.push("log", vec![Dynamic::from(1_i64)]); + }, + true, + ); + agree( + "bump(w); w.level", + |s| { + s.push("w", corpus::Widget::default()); + }, + true, + ); // A constant is not a place rhai will hand out, so the mutation is // discarded — and the caller's entry has to come back untouched. agree( "push(log, 2); log", - |s| { s.push_constant("log", vec![Dynamic::from(1_i64)]); }, + |s| { + s.push_constant("log", vec![Dynamic::from(1_i64)]); + }, true, ); // Read after the other arguments, so it is the second name reported // missing rather than the first. agree("nosuch(gone, missing)", |_| {}, true); - agree("nosuch(gone, brightness)", |s| { s.push("brightness", 1_i64); }, true); + agree( + "nosuch(gone, brightness)", + |s| { + s.push("brightness", 1_i64); + }, + true, + ); } /// A chain rooted at a caller's variable, which is the last root shape that @@ -156,19 +180,67 @@ fn a_chain_can_be_rooted_at_a_caller_variable() { let array = || vec![Dynamic::from(1_i64)]; // A writable entry: read, mutate through a method, and assign through. - agree("host[0]", |s| { s.push("host", array()); }, true); - agree("host.push(2); host", |s| { s.push("host", array()); }, true); - agree("host[0] = 9; host", |s| { s.push("host", array()); }, true); - agree("host.level", |s| { s.push("host", corpus::Widget::default()); }, true); - agree("host.level = 3; host.level", |s| { s.push("host", corpus::Widget::default()); }, true); + agree( + "host[0]", + |s| { + s.push("host", array()); + }, + true, + ); + agree( + "host.push(2); host", + |s| { + s.push("host", array()); + }, + true, + ); + agree( + "host[0] = 9; host", + |s| { + s.push("host", array()); + }, + true, + ); + agree( + "host.level", + |s| { + s.push("host", corpus::Widget::default()); + }, + true, + ); + agree( + "host.level = 3; host.level", + |s| { + s.push("host", corpus::Widget::default()); + }, + true, + ); // A constant is not a place. Rhai refuses the assignment outright and // refuses a mutating method too, because it never hands out a reference to // one and a non-pure native will not take a read-only first argument // (`func/call.rs:405`). - agree("host[0] = 9; host", |s| { s.push_constant("host", array()); }, true); - agree("host.push(2); host", |s| { s.push_constant("host", array()); }, true); - agree("host[0]", |s| { s.push_constant("host", array()); }, true); + agree( + "host[0] = 9; host", + |s| { + s.push_constant("host", array()); + }, + true, + ); + agree( + "host.push(2); host", + |s| { + s.push_constant("host", array()); + }, + true, + ); + agree( + "host[0]", + |s| { + s.push_constant("host", array()); + }, + true, + ); // A shared entry walks through its cell's guard, so the mutation lands // where every holder of the cell can see it. The closure is made in a @@ -176,7 +248,9 @@ fn a_chain_can_be_rooted_at_a_caller_variable() { // two sides render differently on purpose. agree( "{ let keep = || host.len(); } host.push(2); host", - |s| { s.push("host", array()); }, + |s| { + s.push("host", array()); + }, true, ); @@ -237,16 +311,8 @@ fn a_closure_can_capture_a_caller_variable() { // The write happens after the closure is made, so a captured copy answers // with the old length. - agree( - "let n = 0; { let f = || first.len(); first.push(9); n = f.call(); } n", - seed, - true, - ); - agree( - "let n = 0; { let f = || second.len(); second.push(9); n = f.call(); } n", - seed, - true, - ); + agree("let n = 0; { let f = || first.len(); first.push(9); n = f.call(); } n", seed, true); + agree("let n = 0; { let f = || second.len(); second.push(9); n = f.call(); } n", seed, true); // And the capture is what shares it, which `is_shared` can see. agree("{ let f = || first.len(); } is_shared(first)", seed, true); agree("{ let f = || first.len(); } is_shared(second)", seed, true); @@ -258,14 +324,18 @@ fn a_closure_can_capture_a_caller_variable() { fn a_local_shadows_the_caller_without_disturbing_it() { agree( "let brightness = 1; brightness += 1; brightness", - |s| { s.push("brightness", 100_i64); }, + |s| { + s.push("brightness", 100_i64); + }, true, ); // And the other order: read before the local exists, so the same name is // two different variables in one script. agree( "let first = brightness; let brightness = 1; [first, brightness]", - |s| { s.push("brightness", 100_i64); }, + |s| { + s.push("brightness", 100_i64); + }, true, ); } @@ -302,18 +372,12 @@ fn a_name_that_is_nowhere_is_reported_the_same_way() { #[test] fn a_script_function_name_is_not_a_variable() { let engine = corpus::engine(); - let ast = engine - .compile("fn helper() { 1 } let f = helper; f.call()") - .expect("must compile"); + let ast = engine.compile("fn helper() { 1 } let f = helper; f.call()").expect("must compile"); let program = Compiler::new().compile(&ast); + assert!(program.residual_count() > 0, "a function name must not become a name lookup",); assert!( - program.residual_count() > 0, - "a function name must not become a name lookup", - ); - assert!( - !rhaigrain::bytecode::disassemble(program.code()) - .any(|(.., op)| matches!(op, rhaigrain::bytecode::Op::LoadNamed(..))), + !rhai::grain::bytecode::disassemble(program.code()).any(|(.., op)| matches!(op, rhai::grain::bytecode::Op::LoadNamed(..))), "nothing in {:?} may load `helper` by name", program, ); @@ -332,21 +396,14 @@ fn a_global_module_constant_resolves() { let program = Compiler::new().compile(&ast); assert_eq!(program.residual_count(), 0); - let value = Vm::new(&engine) - .run(&program, &mut Scope::new()) - .expect("a module constant must resolve"); + let value = Vm::new(&engine).run(&program, &mut Scope::new()).expect("a module constant must resolve"); assert_eq!(value.as_int().unwrap(), 256); // And writing to one is refused, because it is a value and not a place. let ast = engine.compile("CHANNELS = 1").expect("must compile"); let program = Compiler::new().compile(&ast); - let err = Vm::new(&engine) - .run(&program, &mut Scope::new()) - .expect_err("a module constant is not assignable"); - assert!( - matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), - "got {err:?}", - ); + let err = Vm::new(&engine).run(&program, &mut Scope::new()).expect_err("a module constant is not assignable"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), "got {err:?}",); } /// The first of the three, and the one a VM would most plausibly skip: a @@ -374,26 +431,17 @@ fn a_variable_resolver_is_consulted_first() { scope.push("ordinary", 5_i64); let program = compile(&engine, "injected + ordinary"); - let value = Vm::new(&engine) - .run(&program, &mut scope.clone()) - .expect("both must resolve"); + let value = Vm::new(&engine).run(&program, &mut scope.clone()).expect("both must resolve"); assert_eq!(value.as_int().unwrap(), 104); // A resolver hands back a value rather than a place, so it is read-only. let program = compile(&engine, "injected = 1"); - let err = Vm::new(&engine) - .run(&program, &mut scope.clone()) - .expect_err("a resolved variable is not assignable"); - assert!( - matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), - "got {err:?}", - ); + let err = Vm::new(&engine).run(&program, &mut scope.clone()).expect_err("a resolved variable is not assignable"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), "got {err:?}",); // And the walker agrees about all of it. let ast = engine.compile("injected + ordinary").expect("must compile"); - let expected = engine - .eval_ast_with_scope::(&mut scope.clone(), &ast) - .expect("the walker must agree"); + let expected = engine.eval_ast_with_scope::(&mut scope.clone(), &ast).expect("the walker must agree"); assert_eq!(expected.as_int().unwrap(), 104); } @@ -416,14 +464,10 @@ fn a_resolver_that_grows_the_scope_forces_a_search() { assert_eq!(program.residual_count(), 0); let mut scope = Scope::new(); - let value = Vm::new(&engine) - .run(&program, &mut scope) - .expect("must run"); + let value = Vm::new(&engine).run(&program, &mut scope).expect("must run"); let mut walked = Scope::new(); - let expected = engine - .eval_ast_with_scope::(&mut walked, &ast) - .expect("the walker must run it too"); + let expected = engine.eval_ast_with_scope::(&mut walked, &ast).expect("the walker must run it too"); assert_eq!(format!("{value:?}"), format!("{expected:?}")); } @@ -467,19 +511,9 @@ fn a_resolved_receiver_is_not_the_scope_entry_it_shadows() { // Read-only all the way through, so rhai refuses the call outright rather // than mutating a copy — which is a sharper thing to agree on. - assert!( - matches!( - walker.as_ref().unwrap_err().as_ref(), - rhai::EvalAltResult::ErrorNonPureMethodCallOnConstant(..), - ), - "got {walker:?}", - ); + assert!(matches!(walker.as_ref().unwrap_err().as_ref(), rhai::EvalAltResult::ErrorNonPureMethodCallOnConstant(..),), "got {walker:?}",); assert_eq!(capture(&run, ours), capture(&walked, walker)); - assert_eq!( - format!("{:?}", run.get_value::("shadowed").unwrap()), - "[1]", - "the entry the resolver shadowed must come back untouched", - ); + assert_eq!(format!("{:?}", run.get_value::("shadowed").unwrap()), "[1]", "the entry the resolver shadowed must come back untouched",); } /// The one place a compiled closure is not the walker's closure. @@ -501,22 +535,12 @@ fn a_closure_pointer_is_late_bound() { let program = Compiler::new().compile(&ast); assert_eq!(program.residual_count(), 0, "the closure must lower"); - let ours = Vm::new(&engine) - .run(&program, &mut Scope::new()) - .expect("must run"); - let walker = engine - .eval_ast_with_scope::(&mut Scope::new(), &ast) - .expect("must run under rhai too"); + let ours = Vm::new(&engine).run(&program, &mut Scope::new()).expect("must run"); + let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect("must run under rhai too"); let (ours, walker) = (format!("{ours:?}"), format!("{walker:?}")); - assert!( - ours.starts_with("Fn(\"anon$"), - "ours is a plain named pointer: {ours}", - ); - assert!( - walker.starts_with("Fn*+(\"anon$"), - "rhai's carries a script body and an environment: {walker}", - ); + assert!(ours.starts_with("Fn(\"anon$"), "ours is a plain named pointer: {ours}",); + assert!(walker.starts_with("Fn*+(\"anon$"), "rhai's carries a script body and an environment: {walker}",); // And the difference is only in the binding: calling either gives the // same answer, which is what the corpus covers. @@ -528,20 +552,12 @@ fn a_closure_pointer_is_late_bound() { #[test] fn a_compiled_function_can_be_called_by_name() { let engine = corpus::engine(); - let ast = engine - .compile("fn add(a, b) { a + b } fn boom() { throw 7; } 0") - .expect("must compile"); + let ast = engine.compile("fn add(a, b) { a + b } fn boom() { throw 7; } 0").expect("must compile"); let program = Compiler::new().compile(&ast); let mut vm = Vm::new(&engine); let value = vm - .call_function( - &program, - "add", - vec![Dynamic::from(2_i64), Dynamic::from(3_i64)], - 0, - rhai::Position::NONE, - ) + .call_function(&program, "add", vec![Dynamic::from(2_i64), Dynamic::from(3_i64)], 0, rhai::Position::NONE) .expect("must call"); assert_eq!(value.as_int().unwrap(), 5); @@ -552,23 +568,12 @@ fn a_compiled_function_can_be_called_by_name() { assert!(matches!(*err, rhai::EvalAltResult::ErrorFunctionNotFound(..))); // And what the function raises comes back, wrapped as rhai wraps it. - let err = vm - .call_function(&program, "boom", Vec::new(), 0, rhai::Position::NONE) - .expect_err("must propagate"); - assert!( - matches!(*err, rhai::EvalAltResult::ErrorInFunctionCall(..)), - "got {err:?}", - ); + let err = vm.call_function(&program, "boom", Vec::new(), 0, rhai::Position::NONE).expect_err("must propagate"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorInFunctionCall(..)), "got {err:?}",); // The operand stack is where it started, so a caller can keep using it. let value = vm - .call_function( - &program, - "add", - vec![Dynamic::from(10_i64), Dynamic::from(1_i64)], - 0, - rhai::Position::NONE, - ) + .call_function(&program, "add", vec![Dynamic::from(10_i64), Dynamic::from(1_i64)], 0, rhai::Position::NONE) .expect("must call again"); assert_eq!(value.as_int().unwrap(), 11); } @@ -597,20 +602,12 @@ fn the_compiler_says_whether_a_program_makes_function_pointers() { ] { let ast = engine.compile(source).expect("must compile"); let program = Compiler::new().compile(&ast); - assert_eq!( - program.makes_fn_pointers(), - expected, - "for {source:?}", - ); + assert_eq!(program.makes_fn_pointers(), expected, "for {source:?}",); // Read off the code, so an artifact answers the same. if let Ok(bytes) = program.write() { - let reloaded = rhaigrain::Program::read(&bytes).expect("must load"); - assert_eq!( - reloaded.makes_fn_pointers(), - expected, - "after a round trip, for {source:?}", - ); + let reloaded = rhai::grain::Program::read(&bytes).expect("must load"); + assert_eq!(reloaded.makes_fn_pointers(), expected, "after a round trip, for {source:?}",); } } } @@ -625,14 +622,12 @@ fn a_program_reading_caller_state_can_be_written() { let program = Compiler::new().compile(&ast); let bytes = program.write().expect("must be writable"); - let reloaded = rhaigrain::Program::read(&bytes).expect("must load"); + let reloaded = rhai::grain::Program::read(&bytes).expect("must load"); let mut scope = Scope::new(); scope.push("brightness", 5_i64); scope.push("mode", "chase".to_string()); - let value = Vm::new(&engine) - .run(&reloaded, &mut scope) - .expect("must run"); + let value = Vm::new(&engine).run(&reloaded, &mut scope).expect("must run"); assert_eq!(value.as_int().unwrap(), 15); } diff --git a/tests/mod.rs b/tests/mod.rs new file mode 100644 index 000000000..99648c8e2 --- /dev/null +++ b/tests/mod.rs @@ -0,0 +1,18 @@ +mod grain { + // Declared once here rather than in each harness: they are modules of this + // binary, not crate roots, so a `mod corpus;` of their own would look for + // `/corpus.rs`. + mod corpus; + + // `allocation` is deliberately absent: it owns a counting global allocator + // and is its own binary, declared in Cargo.toml. + mod callback; + mod differential; + mod format; + mod fuzz; + mod limits; + // Prices rhai's own AST nodes, which are exported under `internals` only. + #[cfg(feature = "internals")] + mod projection; + mod scope; +} From 4ec3b7e7bb556db0c9a30655812591c41595b183 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 17:50:17 -0700 Subject: [PATCH 03/33] fix MSRV errors --- src/grain/compile/cases.rs | 6 +++++- src/grain/compile/mod.rs | 3 ++- src/grain/compile/poolable.rs | 6 ++++-- src/grain/vm/mod.rs | 4 +++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/grain/compile/cases.rs b/src/grain/compile/cases.rs index 49a17af9d..b2dc2eb9d 100644 --- a/src/grain/compile/cases.rs +++ b/src/grain/compile/cases.rs @@ -119,7 +119,11 @@ pub(crate) fn split(ranges: &[RangeCase]) -> Vec<(SwitchRange, Vec)> { // Atoms are contiguous and in order, so one that runs the same arms as // the one before it is the same table entry stretched further. - if run.as_ref().is_some_and(|open| open.blocks == blocks) { + if run + .as_ref() + .map(|open| open.blocks == blocks) + .unwrap_or(false) + { let open = run.as_mut().expect("just checked"); open.to = to; open.inclusive = inclusive; diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index d85903f22..d924cc978 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -1212,7 +1212,8 @@ impl Lowering { Expr::DynamicConstant(value, ..) if value .read_lock::() - .is_some_and(|f| f.curry().is_empty()) => + .map(|f| f.curry().is_empty()) + .unwrap_or(false) => { let name = value .read_lock::() diff --git a/src/grain/compile/poolable.rs b/src/grain/compile/poolable.rs index 05a8a9442..31f89ff99 100644 --- a/src/grain/compile/poolable.rs +++ b/src/grain/compile/poolable.rs @@ -32,13 +32,15 @@ pub(crate) fn is_poolable(value: &Dynamic) -> bool { if value.is_array() { return value .read_lock::() - .is_some_and(|array| array.iter().all(is_poolable)); + .map(|array| array.iter().all(is_poolable)) + .unwrap_or(false); } if value.is_map() { return value .read_lock::() - .is_some_and(|map| map.values().all(is_poolable)); + .map(|map| map.values().all(is_poolable)) + .unwrap_or(false); } if value.is_blob() { diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index a2b6387eb..b5d9ab3fc 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -1655,7 +1655,9 @@ impl<'e> Vm<'e> { // (`func/call.rs:1449-1454`), and a function this compiler lowered // copies its first argument whatever it is handed, exactly as rhai // copies it before running a script function (`func/call.rs:661`). - let by_reference = place.is_some_and(|value| !is_shared!(value) && !value.is_read_only()) + let by_reference = place + .map(|value| !is_shared!(value) && !value.is_read_only()) + .unwrap_or(false) && program.function(name_index, argc).is_none(); // All three want the ordinary shape, with every argument on the stack. From 6d7359df0df1e2da4841b7c6817edeedde6e4c73 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 18:56:19 -0700 Subject: [PATCH 04/33] fix review issues --- fuzz/fuzz_targets/generated.rs | 2 +- src/grain/format/mod.rs | 29 ++++++++++++++++++++++++++--- src/grain/pos/varint.rs | 32 +++++++++++++++++++++++++++++--- src/module/mod.rs | 3 +++ 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/fuzz/fuzz_targets/generated.rs b/fuzz/fuzz_targets/generated.rs index 1f274a7a3..9751290f2 100644 --- a/fuzz/fuzz_targets/generated.rs +++ b/fuzz/fuzz_targets/generated.rs @@ -24,7 +24,7 @@ use rhai::{Dynamic, Engine, Scope}; mod generate { include!(concat!( env!("CARGO_MANIFEST_DIR"), - "/../tests/corpus/generate.rs" + "/../tests/grain/corpus/generate.rs" )); } diff --git a/src/grain/format/mod.rs b/src/grain/format/mod.rs index b26138d06..094ed0efa 100644 --- a/src/grain/format/mod.rs +++ b/src/grain/format/mod.rs @@ -193,9 +193,14 @@ impl<'a> Cursor<'a> { let mut value = 0u64; for shift in (0..64).step_by(7) { let byte = self.byte()?; - value |= u64::from(byte & 0x7f) - .checked_shl(shift) - .ok_or(ReadError::MalformedVarint)?; + let payload = u64::from(byte & 0x7f); + // The tenth group has a single bit left to land in. Shifting would + // drop the other six rather than refuse them, so a value too wide + // for 64 bits would decode as a smaller one. + if shift == 63 && payload > 1 { + return Err(ReadError::MalformedVarint); + } + value |= payload << shift; if byte & 0x80 == 0 { return Ok(value); } @@ -307,6 +312,24 @@ mod tests { ); } + /// The tenth group is the one place a shift could silently lose bits, so a + /// wide value there must be refused rather than truncated into a small one. + #[test] + fn a_tenth_group_wider_than_one_bit_is_refused() { + let mut ten = [0x80u8; 10]; + + // The largest value there is: nine full groups and a final bit. + let widest = [[0xffu8; 9].as_slice(), &[0x01]].concat(); + assert_eq!(Cursor::new(&widest).uvarint(), Ok(u64::MAX)); + + // One past it. Shifting would drop the payload and read this as zero. + ten[9] = 0x02; + assert_eq!(Cursor::new(&ten).uvarint(), Err(ReadError::MalformedVarint)); + + ten[9] = 0x7f; + assert_eq!(Cursor::new(&ten).uvarint(), Err(ReadError::MalformedVarint)); + } + #[test] fn reading_past_the_end_is_an_error_not_a_panic() { assert_eq!(Cursor::new(&[]).byte(), Err(ReadError::Truncated)); diff --git a/src/grain/pos/varint.rs b/src/grain/pos/varint.rs index 16526482f..b9d2852c4 100644 --- a/src/grain/pos/varint.rs +++ b/src/grain/pos/varint.rs @@ -29,9 +29,14 @@ pub fn u64(bytes: &[u8], at: &mut usize) -> Result { for shift in (0..64).step_by(7) { let byte = *bytes.get(*at).ok_or(Error::Truncated)?; *at += 1; - value |= ((byte & 0x7f) as u64) - .checked_shl(shift) - .ok_or(Error::Overflow)?; + let payload = u64::from(byte & 0x7f); + // The tenth group has a single bit left to land in. Shifting would drop + // the other six rather than refuse them, so a value too wide for 64 bits + // would decode as a smaller one. + if shift == 63 && payload > 1 { + return Err(Error::Overflow); + } + value |= payload << shift; if byte & 0x80 == 0 { return Ok(value); } @@ -127,6 +132,27 @@ mod tests { assert_eq!(u64(&[0x80], &mut at), Err(Error::Truncated)); } + /// The tenth group is the one place a shift could silently lose bits, so a + /// wide value there must be refused rather than truncated into a small one. + #[test] + fn a_tenth_group_wider_than_one_bit_is_refused() { + let mut ten = [0x80u8; 10]; + + // The largest value there is: nine full groups and a final bit. + ten[9] = 0x01; + let mut at = 0; + assert_eq!(u64(&[[0xffu8; 9].as_slice(), &[0x01]].concat(), &mut at), Ok(u64::MAX)); + + // One past it. Shifting would drop the payload and read this as zero. + ten[9] = 0x02; + let mut at = 0; + assert_eq!(u64(&ten, &mut at), Err(Error::Overflow)); + + ten[9] = 0x7f; + let mut at = 0; + assert_eq!(u64(&ten, &mut at), Err(Error::Overflow)); + } + #[test] fn a_value_too_wide_for_its_field_is_refused() { let mut buf = Vec::new(); diff --git a/src/module/mod.rs b/src/module/mod.rs index cde9a3024..bdb6e06c4 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -374,8 +374,11 @@ impl FuncRegistration { /// # Parameter Examples /// /// `"foo: &str"` <- parameter name = `foo`, type = `&str` + /// /// `"bar"` <- parameter name = `bar`, type unknown + /// /// `"_: i64"` <- parameter name unknown, type = `i64` + /// /// `"MyType"` <- parameter name unknown, type = `MyType` #[cfg(feature = "metadata")] #[must_use] From 3430495bddb720394cdc5186143b89f18466a7cc Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 19:08:47 -0700 Subject: [PATCH 05/33] cargo fmt --- src/grain/bytecode/mod.rs | 8 +- src/grain/format/abi.rs | 15 +- src/grain/pos/varint.rs | 11 +- tests/grain/corpus/generate.rs | 46 +---- tests/grain/corpus/mod.rs | 302 +++++++-------------------------- tests/grain/projection.rs | 30 +--- 6 files changed, 92 insertions(+), 320 deletions(-) diff --git a/src/grain/bytecode/mod.rs b/src/grain/bytecode/mod.rs index e79833c54..d34b6d3d3 100644 --- a/src/grain/bytecode/mod.rs +++ b/src/grain/bytecode/mod.rs @@ -5,17 +5,17 @@ pub mod code; mod chain; mod chunk; -mod strings; -mod switch; mod op; mod positions; +mod strings; +mod switch; mod verify; pub use chain::{Chain, Root, Step, Tail}; -pub use strings::{BadTable, Strings}; -pub use switch::{probe, Switch, SwitchCase, SwitchRange}; pub use chunk::Chunk; pub use code::{assemble, disassemble, resolve_switch_targets, AssembleError, Code}; pub use op::{AssignOp, Op, Receiver}; pub use positions::{Positions, TableError}; +pub use strings::{BadTable, Strings}; +pub use switch::{probe, Switch, SwitchCase, SwitchRange}; pub use verify::{verify, Pools, VerifyError}; diff --git a/src/grain/format/abi.rs b/src/grain/format/abi.rs index f4398ff30..7203e97ad 100644 --- a/src/grain/format/abi.rs +++ b/src/grain/format/abi.rs @@ -93,7 +93,11 @@ impl core::fmt::Display for AbiMismatch { "artifact was written with a {artifact}-byte {what}, but this build has {host}" ), Self::Flag { flag, artifact } => { - let (writer, reader) = if *artifact { ("on", "off") } else { ("off", "on") }; + let (writer, reader) = if *artifact { + ("on", "off") + } else { + ("off", "on") + }; write!( f, "artifact was written with `{flag}` {writer}, but this build has it {reader}" @@ -152,7 +156,9 @@ impl Abi { let bit = differing.trailing_zeros() as usize; // A bit past the table means the writer knew a flag this build does // not. Reporting it as unknown beats indexing out of bounds. - let flag = FLAGS.get(bit).map_or("an unknown restriction", |(name, _)| *name); + let flag = FLAGS + .get(bit) + .map_or("an unknown restriction", |(name, _)| *name); return Some(AbiMismatch::Flag { flag, artifact: self.flags & (1 << bit) != 0, @@ -178,10 +184,7 @@ mod tests { // Against the aliases rather than concrete types, so this holds under // only_i32 and f32_float as well — the point is that the fingerprint // reports the build it was taken on, whichever build that is. - assert_eq!( - abi.int_bytes as usize, - core::mem::size_of::(), - ); + assert_eq!(abi.int_bytes as usize, core::mem::size_of::(),); #[cfg(not(feature = "no_float"))] assert_eq!( abi.float_bytes as usize, diff --git a/src/grain/pos/varint.rs b/src/grain/pos/varint.rs index b9d2852c4..a2133ece7 100644 --- a/src/grain/pos/varint.rs +++ b/src/grain/pos/varint.rs @@ -94,7 +94,11 @@ mod tests { put_u64(&mut buf, value); let mut at = 0; assert_eq!(u64(&buf, &mut at), Ok(value), "at {value}"); - assert_eq!(at, buf.len(), "the reader must consume exactly what was written"); + assert_eq!( + at, + buf.len(), + "the reader must consume exactly what was written" + ); } } @@ -141,7 +145,10 @@ mod tests { // The largest value there is: nine full groups and a final bit. ten[9] = 0x01; let mut at = 0; - assert_eq!(u64(&[[0xffu8; 9].as_slice(), &[0x01]].concat(), &mut at), Ok(u64::MAX)); + assert_eq!( + u64(&[[0xffu8; 9].as_slice(), &[0x01]].concat(), &mut at), + Ok(u64::MAX) + ); // One past it. Shifting would drop the payload and read this as zero. ten[9] = 0x02; diff --git a/tests/grain/corpus/generate.rs b/tests/grain/corpus/generate.rs index 01b60b715..6f5d5748b 100644 --- a/tests/grain/corpus/generate.rs +++ b/tests/grain/corpus/generate.rs @@ -60,11 +60,7 @@ impl Rng { for byte in choices.iter().take(8) { seed = (seed << 8) | u64::from(*byte); } - Self { - choices: choices.to_vec(), - at: 0, - ..Self::new(seed) - } + Self { choices: choices.to_vec(), at: 0, ..Self::new(seed) } } pub fn next(&mut self) -> u64 { @@ -126,9 +122,7 @@ const COMPARISONS: &[&str] = &["==", "!=", "<", "<=", ">", ">="]; /// /// `&&` and `||` are not here — they demand boolean operands at parse time, so /// they belong to [`Generator::condition`] and nowhere else. -const BINARY: &[&str] = &[ - "+", "-", "*", "/", "%", "==", "!=", "<", "<=", ">", ">=", "&", "|", -]; +const BINARY: &[&str] = &["+", "-", "*", "/", "%", "==", "!=", "<", "<=", ">", ">=", "&", "|"]; const METHODS: &[&str] = &["len", "to_string", "abs", "is_empty", "floor", "to_upper"]; @@ -142,20 +136,7 @@ const METHODS: &[&str] = &["len", "to_string", "abs", "is_empty", "floor", "to_u /// Nothing here can grow a value without bound: an argument reaching one of /// these is an arbitrary generated integer, and `pad` would turn that into an /// allocation rather than into a divergence. -const NATIVES: &[(&str, usize)] = &[ - ("push", 2), - ("insert", 3), - ("remove", 2), - ("truncate", 2), - ("reverse", 1), - ("clear", 1), - ("pop", 1), - ("shift", 1), - ("len", 1), - ("is_empty", 1), - ("abs", 1), - ("to_upper", 1), -]; +const NATIVES: &[(&str, usize)] = &[("push", 2), ("insert", 3), ("remove", 2), ("truncate", 2), ("reverse", 1), ("clear", 1), ("pop", 1), ("shift", 1), ("len", 1), ("is_empty", 1), ("abs", 1), ("to_upper", 1)]; pub struct Generator { rng: Rng, @@ -198,10 +179,7 @@ impl Generator { /// `cargo fuzz`. See [`Rng::from_bytes`]. #[must_use] pub fn from_bytes(choices: &[u8]) -> Self { - Self { - rng: Rng::from_bytes(choices), - ..Self::new(0) - } + Self { rng: Rng::from_bytes(choices), ..Self::new(0) } } fn name(&mut self, prefix: &str) -> String { @@ -350,11 +328,7 @@ impl Generator { fn for_loop(&mut self) -> String { let item = self.name("it"); - let iterable = if self.rng.chance(2) { - format!("0..{}", 1 + self.rng.below(4)) - } else { - self.array() - }; + let iterable = if self.rng.chance(2) { format!("0..{}", 1 + self.rng.below(4)) } else { self.array() }; self.vars.push(item.clone()); self.loops += 1; @@ -413,9 +387,7 @@ impl Generator { } fn map(&mut self) -> String { - let entries: Vec = (0..self.rng.below(4)) - .map(|n| format!("k{n}: {}", self.element())) - .collect(); + let entries: Vec = (0..self.rng.below(4)).map(|n| format!("k{n}: {}", self.element())).collect(); format!("#{{ {} }}", entries.join(", ")) } @@ -427,11 +399,7 @@ impl Generator { fn element(&mut self) -> String { if self.depth < MAX_DEPTH && self.rng.chance(4) { self.depth += 1; - let out = if self.rng.chance(2) { - self.array() - } else { - self.map() - }; + let out = if self.rng.chance(2) { self.array() } else { self.map() }; self.depth -= 1; return out; } diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index 0336450a0..953386321 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -41,11 +41,7 @@ pub struct Holder { } fn out_of_range(index: i64, len: usize) -> Box { - Box::new(rhai::EvalAltResult::ErrorArrayBounds( - len, - index, - rhai::Position::NONE, - )) + Box::new(rhai::EvalAltResult::ErrorArrayBounds(len, index, rhai::Position::NONE)) } /// The engine both sides of the differential run against. @@ -54,29 +50,15 @@ pub fn engine() -> rhai::Engine { engine .register_type_with_name::("Widget") - .register_fn("widget", |level: i64| Widget { - level, - cells: vec![10, 20, 30], - }) - .register_get_set( - "level", - |w: &mut Widget| w.level, - |w: &mut Widget, v: i64| w.level = v, - ) + .register_fn("widget", |level: i64| Widget { level, cells: vec![10, 20, 30] }) + .register_get_set("level", |w: &mut Widget| w.level, |w: &mut Widget, v: i64| w.level = v) // Returning an error rather than panicking, because a panic in a // registered function takes the test process with it. .register_indexer_get_set( - |w: &mut Widget, i: i64| -> Result> { - w.cells - .get(i as usize) - .copied() - .ok_or_else(|| out_of_range(i, w.cells.len())) - }, + |w: &mut Widget, i: i64| -> Result> { w.cells.get(i as usize).copied().ok_or_else(|| out_of_range(i, w.cells.len())) }, |w: &mut Widget, i: i64, v: i64| -> Result<(), Box> { let len = w.cells.len(); - *w.cells - .get_mut(i as usize) - .ok_or_else(|| out_of_range(i, len))? = v; + *w.cells.get_mut(i as usize).ok_or_else(|| out_of_range(i, len))? = v; Ok(()) }, ) @@ -89,17 +71,8 @@ pub fn engine() -> rhai::Engine { engine .register_type_with_name::("Holder") - .register_fn("holder", |level: i64| Holder { - inner: Widget { - level, - cells: vec![1, 2, 3], - }, - }) - .register_get_set( - "inner", - |h: &mut Holder| h.inner.clone(), - |h: &mut Holder, w: Widget| h.inner = w, - ); + .register_fn("holder", |level: i64| Holder { inner: Widget { level, cells: vec![1, 2, 3] } }) + .register_get_set("inner", |h: &mut Holder| h.inner.clone(), |h: &mut Holder, w: Widget| h.inner = w); engine } @@ -120,20 +93,13 @@ pub const CASES: &[Case] = &[ // Every segment type goes through a different arm of rhai's rendering: // strings skip dispatch entirely, unit renders empty, and a container // gets its debug-ish form. - case( - "interpolation_of_every_type", - r#"let s = "x"; let n = 1; let f = 1.5; let b = true; let u = (); `${s}|${n}|${f}|${b}|${u}|`"#, - ), - case( - "interpolation_of_containers", - r#"let a = [1, 2]; let m = #{ k: 1 }; `${a}|${m}`"#, - ), + case("interpolation_of_every_type", r#"let s = "x"; let n = 1; let f = 1.5; let b = true; let u = (); `${s}|${n}|${f}|${b}|${u}|`"#), + case("interpolation_of_containers", r#"let a = [1, 2]; let m = #{ k: 1 }; `${a}|${m}`"#), // A host type with no `to_string` registered falls back to the mapped // type name rather than to `Debug`. case("interpolation_of_host_type", r#"let w = widget(3); `w=${w}`"#), case("char_ops", r#"let c = 'a'; c.to_upper()"#), case("unit_value", "()"), - // --- containers ------------------------------------------------------- case("array_literal", "let a = [1, 2, 3]; a[0] + a[1] + a[2]"), case("array_methods", "let a = [3, 1, 2]; a.sort(); a"), @@ -151,12 +117,8 @@ pub const CASES: &[Case] = &[ case("empty_map_nested_in_a_computed_map", "let v = 7; #{ a: v, b: #{}, c: [] }"), // The value is a call, so the order it runs in relative to the rest of the // literal is observable. - case( - "map_computed_order", - r#"let log = ""; fn note(s, c) { s + c } let m = #{ a: note("", "x"), b: note("", "y") }; m.a + m.b"#, - ), + case("map_computed_order", r#"let log = ""; fn note(s, c) { s + c } let m = #{ a: note("", "x"), b: note("", "y") }; m.a + m.b"#), case("nested_containers", r#"let m = #{ xs: [1, 2, #{ y: 3 }] }; m.xs[2].y"#), - // --- control flow ----------------------------------------------------- case("if_else", "let a = 5; if a > 3 { \"big\" } else { \"small\" }"), case("while_loop", "let i = 0; let s = 0; while i < 5 { s += i; i += 1; } s"), @@ -166,107 +128,61 @@ pub const CASES: &[Case] = &[ case("continue_skips", "let s = 0; for i in 0..10 { if i % 2 == 0 { continue; } s += i; } s"), case("for_range", "let s = 0; for i in 0..5 { s += i; } s"), case("for_array", "let s = 0; for x in [10, 20, 30] { s += x; } s"), - case( - "for_with_counter", - "let s = 0; for (x, i) in [10, 20, 30] { s += x * i; } s", - ), + case("for_with_counter", "let s = 0; for (x, i) in [10, 20, 30] { s += x * i; } s"), // The loop variable is pushed once and mutated in place rather than // re-pushed each iteration (eval/stmt.rs:752); a VM that re-pushes would // leave the scope a different depth. - case( - "for_loop_var_not_leaked", - "let x = 99; for x in 0..3 { } x", - ), + case("for_loop_var_not_leaked", "let x = 99; for x in 0..3 { } x"), case("nested_loops_break", "let s = 0; for i in 0..3 { for j in 0..3 { if j == 2 { break; } s += 1; } } s"), // An empty body is a separate path in rhai that never touches the loop // variable or the counter (`eval/stmt.rs:719`). case("for_empty_body", "let s = 0; for i in 0..5 { } s"), // `return` out of a `for` skips the exhaustion path, so the iterator and // both loop variables have to go with the frame. - case( - "for_return_from_body", - "fn find(xs) { for (x, i) in xs { if x > 1 { return i; } } -1 } find([1, 2, 3])", - ), + case("for_return_from_body", "fn find(xs) { for (x, i) in xs { if x > 1 { return i; } } -1 } find([1, 2, 3])"), // A `break` out of a `while` nested in a `for` must drop nothing, and out // of the `for` must drop one — the two are easy to get the wrong way round. - case( - "for_around_while_break", - "let s = 0; for i in 0..3 { let j = 0; while true { j += 1; if j > 2 { break; } s += 1; } } s", - ), + case("for_around_while_break", "let s = 0; for i in 0..3 { let j = 0; while true { j += 1; if j > 2 { break; } s += 1; } } s"), // Iterating a shared cell walks a snapshot, because rhai flattens the // iterable before asking for an iterator (`eval/stmt.rs:677`). - case( - "for_over_captured_array", - "let a = [1, 2, 3]; { let f = || a; } let s = 0; for x in a { s += x; } s", - ), - + case("for_over_captured_array", "let a = [1, 2, 3]; { let f = || a; } let s = 0; for x in a { s += x; } s"), // --- switch ----------------------------------------------------------- case("switch_literal", "let x = 2; switch x { 1 => \"one\", 2 => \"two\", _ => \"other\" }"), case("switch_range", "let x = 42; switch x { 0..=9 => \"small\", 10..=99 => \"medium\", _ => \"large\" }"), // A failing guard must fall through to the next matching case, not to the // default, so both single-digit arms are needed to tell those apart. - case( - "switch_guard", - "let x = 5; switch x { 0..=9 if x % 2 == 1 => \"odd digit\", 0..=9 => \"even digit\", _ => \"big\" }", - ), + case("switch_guard", "let x = 5; switch x { 0..=9 if x % 2 == 1 => \"odd digit\", 0..=9 => \"even digit\", _ => \"big\" }"), case("switch_default_only", "switch 999 { 1 => \"a\", _ => \"fallback\" }"), // Two case values, one arm: the table has two entries pointing at one // body, which a compiler emitting a body per entry would duplicate. - case( - "switch_shared_body", - "let x = 2; switch x { 1 | 2 => \"low\", 3 => \"three\", _ => \"other\" }", - ), + case("switch_shared_body", "let x = 2; switch x { 1 | 2 => \"low\", 3 => \"three\", _ => \"other\" }"), // The rule that reads like a bug and is not: a case value that matched but // whose guard declined goes to the *default*, never on to the ranges // (eval/stmt.rs:544). Without the range arm here the two are the same // answer and the case proves nothing. - case( - "switch_declined_case_skips_ranges", - "let f = false; let x = 1; switch x { 1 if f => \"guarded\", 0..=5 => \"range\", _ => \"default\" }", - ), + case("switch_declined_case_skips_ranges", "let f = false; let x = 1; switch x { 1 if f => \"guarded\", 0..=5 => \"range\", _ => \"default\" }"), // No `_` arm at all, so the miss has to produce unit from nowhere. case("switch_no_default", "let x = 9; switch x { 1 => \"a\" }"), - case( - "switch_string", - "let s = \"b\"; switch s { \"a\" => 1, \"b\" => 2, _ => 0 }", - ), + case("switch_string", "let s = \"b\"; switch s { \"a\" => 1, \"b\" => 2, _ => 0 }"), // A range arm covers the reals between its bounds, so a float lands in one // even though the bounds are integers. - case( - "switch_float_in_range", - "let x = 5.5; switch x { 0..10 => \"in\", _ => \"out\" }", - ), + case("switch_float_in_range", "let x = 5.5; switch x { 0..10 => \"in\", _ => \"out\" }"), // Hashing a host type panics, so the subject has to be checked before it // reaches a hasher — and must still find the default. - case( - "switch_unhashable_subject", - "let w = widget(3); switch w { 1 => \"int\", _ => \"other\" }", - ), + case("switch_unhashable_subject", "let w = widget(3); switch w { 1 => \"int\", _ => \"other\" }"), // A shared value is not hashable either, so rhai skips the cases *and* the // ranges and goes straight to the default — however well the value would // otherwise have matched. Reading the subject through its cell would hide // that, which is why the subject is loaded unflattened. - case( - "switch_on_a_shared_subject_takes_the_default", - r#"let v = 0; { let f = || v; } switch v { 0 => "case", _ => "default" }"#, - ), - case( - "switch_range_on_a_shared_subject_takes_the_default", - r#"let v = 5; { let f = || v; } switch v { 0..=9 => "range", _ => "default" }"#, - ), + case("switch_on_a_shared_subject_takes_the_default", r#"let v = 0; { let f = || v; } switch v { 0 => "case", _ => "default" }"#), + case("switch_range_on_a_shared_subject_takes_the_default", r#"let v = 5; { let f = || v; } switch v { 0..=9 => "range", _ => "default" }"#), // The same shape before anything shares it, so the pair says the difference // is the sharing rather than the switch. - case( - "switch_on_an_unshared_subject_matches", - r#"let v = 0; switch v { 0 => "case", _ => "default" }"#, - ), + case("switch_on_an_unshared_subject_matches", r#"let v = 0; switch v { 0 => "case", _ => "default" }"#), // An arm body is a block: it declares, and it has to leave the scope the // depth it found it — which only shows up in something that reads a local // afterwards. - case( - "switch_block_body_scope", - "let x = 1; let y = 0; switch x { 1 => { let z = 5; y = z * 2 }, _ => () } y", - ), + case("switch_block_body_scope", "let x = 1; let y = 0; switch x { 1 => { let z = 5; y = z * 2 }, _ => () } y"), // A jump out of an arm and out of the switch, which is where the operand // stack most plausibly ends up a different depth on the two paths. // A range is a host type as far as `Dynamic` is concerned, and @@ -274,41 +190,23 @@ pub const CASES: &[Case] = &[ // fact hash a range (types/dynamic.rs:465). So rhai never matches a range // *subject* against anything, and neither may the VM: mirroring the gate // matters more than being clever about it. - case( - "switch_range_subject_never_matches", - "let r = 0..5; switch r { 0..5 => \"same\", _ => \"no\" }", - ), - case( - "switch_break_from_loop", - "let s = 0; let i = 0; while i < 10 { switch i { 3 => break, _ => () } s += 1; i += 1; } s", - ), - + case("switch_range_subject_never_matches", "let r = 0..5; switch r { 0..5 => \"same\", _ => \"no\" }"), + case("switch_break_from_loop", "let s = 0; let i = 0; while i < 10 { switch i { 3 => break, _ => () } s += 1; i += 1; } s"), // --- blocks used for their value --------------------------------------- // Rhai wraps a block in `Expr::Stmt` wherever a value is wanted, so these // are one construct in three disguises. Each declares inside the block, so // a lowering that forgot to rewind would leave the scope a different depth // and every slot after it would name the wrong variable. - case( - "let_from_switch", - "let x = 2; let y = switch x { 1 => \"one\", 2 => \"two\", _ => \"other\" }; y", - ), - case( - "let_from_if", - "let c = true; let y = if c { let a = 1; a } else { let b = 2; b }; y + 10", - ), + case("let_from_switch", "let x = 2; let y = switch x { 1 => \"one\", 2 => \"two\", _ => \"other\" }; y"), + case("let_from_if", "let c = true; let y = if c { let a = 1; a } else { let b = 2; b }; y + 10"), case("let_from_block", "let a = 3; let y = { let z = a; z * 2 }; y"), // A block among a call's arguments, where the scope grows while operands // are already on the stack. - case( - "block_as_argument", - "fn add(a, b) { a + b } let n = 2; add({ let t = n; t + 1 }, 10)", - ), - + case("block_as_argument", "fn add(a, b) { a + b } let n = 2; add({ let t = n; t + 1 }, 10)"), // --- scoping ---------------------------------------------------------- case("shadowing_nested", "let x = 1; { let x = 2; { let x = 3; } } x"), case("block_scope_discarded", "let x = 1; { let y = 2; x += y; } x"), case("const_read", "const K = 10; K * 2"), - // --- functions -------------------------------------------------------- case("fn_call", "fn add(a, b) { a + b } add(2, 3)"), // Kept shallow deliberately: rhai's default call-depth limit is far lower @@ -329,11 +227,7 @@ pub const CASES: &[Case] = &[ // // `this` is what leaves the body unlowerable, and calling it by name // rather than as a method is what routes it through generic dispatch. - case( - "error_a_skipped_function_cannot_see_the_caller", - "fn peek(k) { let seen = secret; this } let secret = 42; peek(1)", - ), - + case("error_a_skipped_function_cannot_see_the_caller", "fn peek(k) { let seen = secret; this } let secret = 42; peek(1)"), // --- a variable in first-argument position ------------------------------ // Rhai rewrites `f(x, ..)` into `x.f(..)` so a `&mut` first parameter // mutates the variable (`func/call.rs:1434`). These are the same calls the @@ -344,14 +238,8 @@ pub const CASES: &[Case] = &[ // Rhai reads the variable *after* the other arguments, so an argument that // writes to it is seen. Two shapes of write, because one goes through the // rewrite itself and the other does not. - case( - "call_style_argument_writes_the_receiver", - "let a = [1]; push(a, { push(a, 9); 2 }); a", - ), - case( - "call_style_argument_replaces_the_receiver", - "let a = [1]; push(a, { a = [7]; 2 }); a", - ), + case("call_style_argument_writes_the_receiver", "let a = [1]; push(a, { push(a, 9); 2 }); a"), + case("call_style_argument_replaces_the_receiver", "let a = [1]; push(a, { a = [7]; 2 }); a"), // The receiver appearing again among the arguments, which is where a live // reference into the scope would be most likely to show. It does not: the // later argument was read and flattened before the reference was taken, so @@ -364,10 +252,7 @@ pub const CASES: &[Case] = &[ // The closure is made in a block so the scope the two sides are compared on // does not end up holding a pointer, which they render differently on // purpose — see `a_closure_pointer_is_late_bound` in `tests/scope.rs`. - case( - "call_style_shared_receiver", - "let a = [1]; { let f = || a.len(); } push(a, 2); a", - ), + case("call_style_shared_receiver", "let a = [1]; { let f = || a.len(); } push(a, 2); a"), // A script function copies its first argument whichever way it arrives, so // the rewrite is invisible here — which is the thing to pin. case("call_style_script_fn", "fn bump_it(x) { x += 1; x } let n = 3; bump_it(n); n"), @@ -382,7 +267,6 @@ pub const CASES: &[Case] = &[ // shapes, because one goes through the rewrite and the other does not. case("error_returned_by_a_native", r#"parse_int("zz")"#), case("error_returned_by_a_native_by_reference", r#"let s = "zz"; parse_int(s)"#), - // --- closures --------------------------------------------------------- // Function pointers are invoked via `.call()`; `f(5)` would look for a // function literally named `f`. @@ -392,37 +276,19 @@ pub const CASES: &[Case] = &[ // difference is the price of not shipping an AST body, it is script- // visible, and `a_closure_pointer_is_late_bound` is where it is pinned — // so these cases test the capture rather than re-testing the rendering. - case( - "closure_capture_read", - "let n = 10; let r = 0; { let f = |x| x + n; r = f.call(5); } r", - ), + case("closure_capture_read", "let n = 10; let r = 0; { let f = |x| x + n; r = f.call(5); } r"), // Capture is by shared cell, so the mutation must be visible outside. - case( - "closure_capture_mutate", - "let n = 0; { let f = || n += 1; f.call(); f.call(); } n", - ), + case("closure_capture_mutate", "let n = 0; { let f = || n += 1; f.call(); f.call(); } n"), // In a block for the same reason the closure cases are: what the pointer // *does* matches, what it renders as does not. - case( - "fn_ptr_call", - "fn triple(x) { x * 3 } let r = 0; { let f = Fn(\"triple\"); r = f.call(4); } r", - ), + case("fn_ptr_call", "fn triple(x) { x * 3 } let r = 0; { let f = Fn(\"triple\"); r = f.call(4); } r"), // The same through a name that is not a constant, so rhai's optimizer // cannot fold it into a pointer carrying an environment. - case( - "fn_ptr_from_dynamic_name", - "fn triple(x) { x * 3 } let n = \"trip\" + \"le\"; let f = Fn(n); f.call(4)", - ), - case( - "fn_ptr_curried", - "fn add(a, b) { a + b } let n = \"a\" + \"dd\"; let f = Fn(n).curry(10); f.call(5)", - ), + case("fn_ptr_from_dynamic_name", "fn triple(x) { x * 3 } let n = \"trip\" + \"le\"; let f = Fn(n); f.call(4)"), + case("fn_ptr_curried", "fn add(a, b) { a + b } let n = \"a\" + \"dd\"; let f = Fn(n).curry(10); f.call(5)"), // A pointer to a native function goes to rhai's own dispatch rather than // to a chunk of ours. - case( - "fn_ptr_to_native", - "let n = \"ab\" + \"s\"; let f = Fn(n); f.call(-7)", - ), + case("fn_ptr_to_native", "let n = \"ab\" + \"s\"; let f = Fn(n); f.call(-7)"), // Deliberately absent: `let x = 1; x.call(2)`. That is not an error in // rhai — a non-pointer target means the *argument* is the pointer and the // target is `this` — and the VM reproduces the behaviour, but not the @@ -431,10 +297,7 @@ pub const CASES: &[Case] = &[ // was measured to move the divergence onto the common path instead of // removing it. A pool of positions would fix it and would not be // strippable. - case( - "error_fn_ptr_unknown_name", - "let n = \"no\" + \"pe\"; let f = Fn(n); f.call(1)", - ), + case("error_fn_ptr_unknown_name", "let n = \"no\" + \"pe\"; let f = Fn(n); f.call(1)"), // `Fn` and `curry` read their first argument and blame everything they can // then complain about on *it* rather than on the call — a name that is not // a string, a string that is not an identifier, a first argument that is @@ -447,30 +310,17 @@ pub const CASES: &[Case] = &[ // instead of writing *through* it severs the closure silently — the value // is right and the aliasing is dead — so this needs a write after the // capture to catch it. - case( - "closure_shared_write", - "let x = 1; let r = 0; { let f = || x; x = 42; r = f.call(); } r", - ), - case( - "closure_shared_op_assign", - "let x = 1; let r = 0; { let f = || x; x += 41; r = f.call(); } r", - ), + case("closure_shared_write", "let x = 1; let r = 0; { let f = || x; x = 42; r = f.call(); } r"), + case("closure_shared_op_assign", "let x = 1; let r = 0; { let f = || x; x += 41; r = f.call(); } r"), // The same cell as the root of a chain. `get_indexed_mut` refuses a shared // value outright, so walking one takes the host down rather than returning // an error (`eval/chaining.rs:461`). - case( - "closure_shared_chain_root", - "let a = [1, 2, 3]; { let f = || a[0]; } a[1] = 20; a[1]", - ), + case("closure_shared_chain_root", "let a = [1, 2, 3]; { let f = || a[0]; } a[1] = 20; a[1]"), // rhai answers this syntactically and registers no function for it, so a // lowered call would fail to resolve where the walker returns a bool. - case( - "is_shared_after_capture", - "let x = 1; let r = false; { let f = || x; r = is_shared(f); } [is_shared(x), r]", - ), + case("is_shared_after_capture", "let x = 1; let r = false; { let f = || x; r = is_shared(f); } [is_shared(x), r]"), case("closure_in_map", "[1, 2, 3].map(|x| x * 2)"), case("closure_in_filter", "[1, 2, 3, 4].filter(|x| x % 2 == 0)"), - // --- chained lvalues -------------------------------------------------- // Each of these needs a different `Target` variant and its write-back. case("index_assign_array", "let a = [1, 2, 3]; a[1] = 99; a"), @@ -487,10 +337,7 @@ pub const CASES: &[Case] = &[ case("error_map_write_through_an_absent_key", "let m = #{}; m.a.b = 1; m"), // A closure holds the same cell, so a key invented by a read would be // visible from outside the expression that invented it. - case( - "map_read_of_absent_key_is_not_visible_to_a_closure", - "let m = #{}; let r = 0; { let f = || m; r = m.zz; } [m, r]", - ), + case("map_read_of_absent_key_is_not_visible_to_a_closure", "let m = #{}; let r = 0; { let f = || m; r = m.zz; } [m, r]"), case("op_assign_indexed", "let a = [1, 2, 3]; a[0] += 10; a"), case("bitfield_assign", "let x = 0; x[2] = true; x"), case("string_char_assign", r#"let s = "hello"; s[0] = 'H'; s"#), @@ -499,7 +346,6 @@ pub const CASES: &[Case] = &[ // one does not cover the other. A string rather than an array, because // rhai indexes arrays with integers only and slices them with `extract`. case("string_slice_inclusive", r#"let s = "hello world"; s[6..=9]"#), - // --- chains rooted at something that is not a variable ------------------ // Rhai evaluates the root into a temporary and walks that // (`eval/chaining.rs:561-571`), so there is no scope entry behind it and @@ -543,34 +389,23 @@ pub const CASES: &[Case] = &[ // yields a bool, and rhai names the *second* `[`. case("error_index_into_an_unindexable_step", "let n = 0; n[0][5]"), case("error_index_into_an_unindexable_step_deep", "let m = #{ a: 1 }; m.a[0][5]"), - // --- what an escaping error leaves in the scope ------------------------- // Rhai rewinds a block whether it is left normally or by a throw, and // rewinds nothing at the top level. The comparison that matters in all of // these is the leftover scope rather than the error. case("throw_from_a_block_rewinds_it", "let a = 1; { let b = 2; throw 3; }"), case("throw_from_a_for_body_drops_the_loop_var", "let a = 1; for i in 0..3 { throw i; }"), - case( - "throw_from_a_while_body_drops_its_locals", - "let a = 1; let n = 0; while n < 3 { let b = n; throw b; }", - ), + case("throw_from_a_while_body_drops_its_locals", "let a = 1; let n = 0; while n < 3 { let b = n; throw b; }"), // A catch block is a block too, and its variable is the one rhai pushes // rather than the script. case("throw_from_a_catch_drops_the_catch_var", "let a = 1; try { throw 2; } catch (e) { throw e; }"), - case( - "throw_from_a_nested_block_drops_every_level", - "let a = 1; { let b = 2; { let c = 3; for i in 0..2 { throw i; } } }", - ), + case("throw_from_a_nested_block_drops_every_level", "let a = 1; { let b = 2; { let c = 3; for i in 0..2 { throw i; } } }"), // The frame boundary: a function's own locals go with its scope, and the // caller's top-level ones stay. - case( - "throw_from_a_function_leaves_the_caller_top_level_alone", - "fn boom() { let inner = 9; throw inner; } let a = 1; { let b = 2; boom(); }", - ), + case("throw_from_a_function_leaves_the_caller_top_level_alone", "fn boom() { let inner = 9; throw inner; } let a = 1; { let b = 2; boom(); }"), // Nothing to rewind, which is the case a floor set too low would break. case("throw_at_the_top_level_keeps_what_ran", "let a = 1; let b = 2; throw 3;"), case("error_temp_root_out_of_bounds", "[1, 2, 3][99]"), - // --- errors ----------------------------------------------------------- // Compared by variant and position, so a VM that reports the right failure // at the wrong place still fails the test. @@ -587,20 +422,16 @@ pub const CASES: &[Case] = &[ // it up. case("error_condition_is_a_range", "if 0..1 { 1 } else { 2 }"), case("error_condition_is_a_host_type", "let w = widget(1); while w { 1 }"), - case("error_type_mismatch",r#"let a = 1; a + "string" + [1]"#), + case("error_type_mismatch", r#"let a = 1; a + "string" + [1]"#), case("error_divide_by_zero", "let a = 1; a / 0"), case("throw_value", "throw 42"), case("throw_in_fn", "fn f() { throw \"boom\"; } f()"), - // --- try / catch ------------------------------------------------------ case("try_catch_value", "try { throw 7; } catch (e) { e * 2 }"), case("try_catch_native_error", "try { let a = [1]; a[9] } catch (e) { e.message != () }"), case("try_catch_rethrow", "try { try { throw 1; } catch { throw; } } catch (e) { e }"), // `return` unwinds as an error but must pass straight through a catch. - case( - "try_catch_does_not_swallow_return", - "fn f() { try { return 1; } catch { return 2; } } f()", - ), + case("try_catch_does_not_swallow_return", "fn f() { try { return 1; } catch { return 2; } } f()"), case("try_catch_no_error", "try { 5 } catch { 6 }"), // The catch block's value is discarded — the statement is unit on the // caught path and the try block's value otherwise (`eval/stmt.rs:863`). @@ -608,31 +439,15 @@ pub const CASES: &[Case] = &[ // A jump out of a `try` skips the `PopHandler` the straight-line path // would have run. Left armed, the next error is caught into a block that // has already been left — so the second failure here must not be caught. - case( - "break_out_of_try_disarms_it", - "let s = 0; while true { try { throw 1; } catch { break; } } try { throw 2; } catch (e) { s = e; } s", - ), - case( - "break_out_of_for_inside_try", - "let s = 0; for i in 0..5 { try { if i == 2 { break; } s += i; } catch { s = -1; } } s", - ), - case( - "continue_out_of_try_inside_for", - "let s = 0; for i in 0..5 { try { if i % 2 == 0 { continue; } s += i; } catch { s = -1; } } s", - ), + case("break_out_of_try_disarms_it", "let s = 0; while true { try { throw 1; } catch { break; } } try { throw 2; } catch (e) { s = e; } s"), + case("break_out_of_for_inside_try", "let s = 0; for i in 0..5 { try { if i == 2 { break; } s += i; } catch { s = -1; } } s"), + case("continue_out_of_try_inside_for", "let s = 0; for i in 0..5 { try { if i % 2 == 0 { continue; } s += i; } catch { s = -1; } } s"), // An error out of a called function arrives wrapped in // `ErrorInFunctionCall`, which is catchable, and `unwrap_inner` is what // still binds the bare thrown value. - case( - "try_around_a_compiled_call", - "fn boom() { throw 7; } try { boom(); } catch (e) { e }", - ), + case("try_around_a_compiled_call", "fn boom() { throw 7; } try { boom(); } catch (e) { e }"), // `return` is a pseudo error and must pass straight through a handler. - case( - "try_does_not_catch_return", - "fn f() { try { return 1; } catch { 2 } } f()", - ), - + case("try_does_not_catch_return", "fn f() { try { return 1; } catch { 2 } } f()"), // --- host types ------------------------------------------------------- // // The one part of the chain walker that approximates rather than @@ -649,10 +464,7 @@ pub const CASES: &[Case] = &[ case("host_method_pure", "let w = widget(4); w.doubled()"), // Two levels, so the middle one is a temporary. case("host_temp_set", "let h = holder(3); h.inner.level = 8; h.inner.level"), - case( - "host_temp_index_set", - "let h = holder(3); h.inner[0] = 7; h.inner[0]", - ), + case("host_temp_index_set", "let h = holder(3); h.inner[0] = 7; h.inner[0]"), // A mutating call on a temporary: rhai writes it back, so the change // survives. case("host_temp_mutates", "let h = holder(3); h.inner.bump(); h.inner.level"), diff --git a/tests/grain/projection.rs b/tests/grain/projection.rs index 4152b2620..69f99244f 100644 --- a/tests/grain/projection.rs +++ b/tests/grain/projection.rs @@ -164,12 +164,9 @@ fn interned_names(ast: &AST) -> BTreeSet { #[test] fn projected_artifact_size() { let engine = Engine::new(); - let ast = engine - .compile(SOURCE) - .expect("fixtures/follow.rhai must compile"); + let ast = engine.compile(SOURCE).expect("fixtures/follow.rhai must compile"); - let mut counts: std::collections::BTreeMap<&'static str, (usize, usize)> = - std::collections::BTreeMap::new(); + let mut counts: std::collections::BTreeMap<&'static str, (usize, usize)> = std::collections::BTreeMap::new(); let mut code_bytes = 0usize; let mut nodes = 0usize; @@ -193,11 +190,7 @@ fn projected_artifact_size() { // A tag byte plus eight bytes of payload, for every node that referenced // the pool. Deduplication would shrink this; not modelling it keeps the // projection on the pessimistic side. - let pool_refs: usize = ["int", "float", "string", "char", "dynamic const"] - .iter() - .filter_map(|k| counts.get(k)) - .map(|(n, _)| *n) - .sum(); + let pool_refs: usize = ["int", "float", "string", "char", "dynamic const"].iter().filter_map(|k| counts.get(k)).map(|(n, _)| *n).sum(); let const_pool = pool_refs * 9; // Header, ABI fingerprint, and section offsets. @@ -213,26 +206,15 @@ fn projected_artifact_size() { } println!("\n{:<18} {:>9}", "code", code_bytes); - println!( - "{:<18} {:>9} ({} distinct names)", - "string table", - string_table, - names.len() - ); + println!("{:<18} {:>9} ({} distinct names)", "string table", string_table, names.len()); println!("{:<18} {:>9} ({pool_refs} pool refs)", "constants", const_pool); println!("{:<18} {:>9}", "header", HEADER); println!("{:<18} {:>9}", "projected total", projected); - println!( - "\nprojected artifact / source {:.2}x", - projected as f64 / source_bytes as f64 - ); + println!("\nprojected artifact / source {:.2}x", projected as f64 / source_bytes as f64); assert!(nodes > 0, "the walk visited nothing"); - assert!( - code_bytes > 0, - "every node priced at zero means the model is broken", - ); + assert!(code_bytes > 0, "every node priced at zero means the model is broken",); assert!( !counts.contains_key(UNPRICED), "{} nodes have no cost in the model, so the projection understates: \ From 14be485399198aa9acaf2f10e038de2ba43bef07 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 19:18:29 -0700 Subject: [PATCH 06/33] fix semi-unused import, add tests to CI, fix test naming --- .github/workflows/build.yml | 4 +++- Cargo.toml | 4 ++-- src/eval/chaining.rs | 16 ++++++++-------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa7e75cea..bc6f59316 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: check - args: "--features decimal,metadata,serde,debugging" + args: "--features decimal,metadata,serde,debugging,grain" # typical build with various feature combinations build: @@ -46,6 +46,7 @@ jobs: os: [ubuntu-latest] flags: - "" + - "--features grain" - "--features testing-environ,debugging" - "--features testing-environ,metadata" - "--features testing-environ,serde" @@ -62,6 +63,7 @@ jobs: - "--features testing-environ,f32_float,serde,metadata,internals,debugging" - "--features testing-environ,no_custom_syntax,serde,metadata,internals,debugging" - "--tests --features testing-environ,only_i32,serde,metadata,internals,debugging" + - "--tests --features testing-environ,only_i32,serde,metadata,internals,debugging,grain" - "--features testing-environ,only_i64,serde,metadata,internals,debugging" - "--features testing-environ,no_index,serde,metadata,internals,debugging" - "--features testing-environ,no_object,serde,metadata,internals,debugging" diff --git a/Cargo.toml b/Cargo.toml index f7ae7f380..c7708e476 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,7 +152,7 @@ required-features = ["metadata", "internals"] # The grain harnesses, as one binary. `tests/mod.rs` wraps them in an inline # `mod grain`, which is what puts their module paths inside `tests/grain/`. [[test]] -name = "mod" +name = "grain" path = "tests/mod.rs" required-features = ["grain"] @@ -160,7 +160,7 @@ required-features = ["grain"] # and reads process-global counters as deltas around each call, so any test # allocating on another thread corrupts every number it reports. [[test]] -name = "grain_allocation" +name = "grain_allocation_efficiency" path = "tests/grain/allocation.rs" required-features = ["grain"] diff --git a/src/eval/chaining.rs b/src/eval/chaining.rs index d90d64923..be16346ab 100644 --- a/src/eval/chaining.rs +++ b/src/eval/chaining.rs @@ -6,8 +6,8 @@ use crate::ast::{ASTFlags, BinaryExpr, Expr, OpAssignment}; use crate::engine::{FN_IDX_GET, FN_IDX_SET}; use crate::types::dynamic::Union; use crate::{ - calc_fn_hash, Dynamic, Engine, ExclusiveRange, FnArgsVec, InclusiveRange, OnceCell, Position, - RhaiResult, RhaiResultOf, Scope, ERR, + calc_fn_hash, Dynamic, Engine, FnArgsVec, OnceCell, Position, RhaiResult, RhaiResultOf, Scope, + ERR, }; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -354,8 +354,8 @@ impl Engine { // Range index on empty string - empty slice Err(typ) - if (typ == std::any::type_name::() - || typ == std::any::type_name::()) + if (typ == std::any::type_name::() + || typ == std::any::type_name::()) && s.is_empty() => { let value = s.clone().into(); @@ -369,9 +369,9 @@ impl Engine { } // Range index - slice - Err(typ) if typ == std::any::type_name::() => { + Err(typ) if typ == std::any::type_name::() => { // val_str[range] - let range = idx.read_lock::().unwrap().clone(); + let range = idx.read_lock::().unwrap().clone(); let chars_count = s.chars().count(); let start = if range.start >= 0 { @@ -405,9 +405,9 @@ impl Engine { exclusive: true, }) } - Err(typ) if typ == std::any::type_name::() => { + Err(typ) if typ == std::any::type_name::() => { // val_str[range] - let range = idx.read_lock::().unwrap().clone(); + let range = idx.read_lock::().unwrap().clone(); let chars_count = s.chars().count(); let start = if *range.start() >= 0 { From a95baff49d12499f12c360de5e5f716337a8f3b4 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 19:54:47 -0700 Subject: [PATCH 07/33] cleanup, fix CI, add examples --- src/grain/bytecode/op.rs | 4 +-- src/grain/bytecode/switch.rs | 41 +++++++++++++++++------- src/grain/compile/mod.rs | 7 ++-- src/grain/compile/poolable.rs | 16 +++++----- src/grain/format/abi.rs | 13 +++++--- src/grain/format/write.rs | 4 +-- src/grain/mod.rs | 60 ++++++++++++++++++++++++++++++++++- tests/grain/allocation.rs | 40 ++++++++++++----------- tests/grain/callback.rs | 2 +- tests/grain/corpus/mod.rs | 23 ++++++++------ tests/grain/differential.rs | 2 +- tests/grain/format.rs | 19 ++++++----- tests/grain/projection.rs | 15 +++++---- tests/grain/scope.rs | 54 ++++++++++++++++--------------- 14 files changed, 197 insertions(+), 103 deletions(-) diff --git a/src/grain/bytecode/op.rs b/src/grain/bytecode/op.rs index e4c03b5c9..006375a1c 100644 --- a/src/grain/bytecode/op.rs +++ b/src/grain/bytecode/op.rs @@ -59,8 +59,8 @@ pub enum Receiver { /// A stack machine: operands are pushed and consumed on an operand stack, and /// locals live in slots addressed directly. `EvalAst` is the escape hatch that /// hands a fragment back to rhai's tree walker, so anything the compiler cannot -/// yet lower still runs, and the whole language stays covered. Milestones -/// convert residuals into instructions rather than adding coverage. +/// yet lower still runs, and the whole language stays covered. Lowering more of +/// it converts residuals into instructions rather than adding coverage. /// /// Instructions carry no source position. Several of them can fail against a /// place in the source, and the position for that comes from the program's diff --git a/src/grain/bytecode/switch.rs b/src/grain/bytecode/switch.rs index defc69c9d..182e81d46 100644 --- a/src/grain/bytecode/switch.rs +++ b/src/grain/bytecode/switch.rs @@ -144,6 +144,15 @@ fn case_hash(value: &Dynamic) -> Option { mod tests { use super::*; + /// A script integer, as a subject or a case. + /// + /// Spelled through `INT` rather than as an `i64` literal because `only_i32` + /// narrows it: a `Dynamic` built from the wider type there is a boxed host + /// value, which has no hash and so matches nothing. + fn int(value: INT) -> Dynamic { + Dynamic::from(value) + } + /// A host type, which has no hash. #[derive(Debug, Clone)] struct Opaque; @@ -166,21 +175,29 @@ mod tests { #[test] fn a_matching_case_wins() { - let (one, two) = (Dynamic::from(1_i64), Dynamic::from(2_i64)); + let (one, two) = (int(1), int(2)); let table = table(&[(&one, 10), (&two, 20)], Vec::new(), 99); assert_eq!(table.dispatch(&one), 10); assert_eq!(table.dispatch(&two), 20); - assert_eq!(table.dispatch(&Dynamic::from(3_i64)), 99); + assert_eq!(table.dispatch(&int(3)), 99); } /// The distinction that makes this hashing rather than `==`: rhai does not /// match an integer against a float case, even though `1 == 1.0`. + #[cfg(not(feature = "no_float"))] #[test] fn a_float_does_not_match_an_integer_case() { - let one = Dynamic::from(1_i64); + let one = int(1); let table = table(&[(&one, 10)], Vec::new(), 99); - assert_eq!(table.dispatch(&Dynamic::from(1.0_f64)), 99); + + let float = Dynamic::from(1.0 as crate::FLOAT); + // The subject has to reach the hasher for this to say anything. Built + // from a literal `f64` it would not under `f32_float`; that is a boxed + // host value, and it would land on the default for having no hash at + // all rather than for hashing differently. + assert!(float.is_hashable(), "this test needs a hashable float"); + assert_eq!(table.dispatch(&float), 99); } #[test] @@ -229,7 +246,7 @@ mod tests { /// Ranges are consulted only after the cases miss. #[test] fn a_range_catches_what_no_case_did() { - let one = Dynamic::from(1_i64); + let one = int(1); let table = table( &[(&one, 10)], vec![ @@ -250,18 +267,18 @@ mod tests { ); assert_eq!(table.dispatch(&one), 10, "a case still wins"); - assert_eq!(table.dispatch(&Dynamic::from(5_i64)), 20); - assert_eq!(table.dispatch(&Dynamic::from(7_i64)), 20); - assert_eq!(table.dispatch(&Dynamic::from(8_i64)), 30, "exclusive end"); - assert_eq!(table.dispatch(&Dynamic::from(10_i64)), 30, "inclusive end"); - assert_eq!(table.dispatch(&Dynamic::from(11_i64)), 99); + assert_eq!(table.dispatch(&int(5)), 20); + assert_eq!(table.dispatch(&int(7)), 20); + assert_eq!(table.dispatch(&int(8)), 30, "exclusive end"); + assert_eq!(table.dispatch(&int(10)), 30, "inclusive end"); + assert_eq!(table.dispatch(&int(11)), 99); } /// Hashing one would panic, so it must never reach the hasher — and it /// must still be able to reach the default. #[test] fn an_unhashable_subject_falls_through_rather_than_panicking() { - let one = Dynamic::from(1_i64); + let one = int(1); let table = table(&[(&one, 10)], Vec::new(), 99); // A bare function pointer *is* hashable; only one carrying an @@ -277,7 +294,7 @@ mod tests { #[test] fn an_unhashable_case_has_no_hash_to_key_on() { assert_eq!(case_hash(&Dynamic::from(Opaque)), None); - assert!(case_hash(&Dynamic::from(1_i64)).is_some()); + assert!(case_hash(&int(1)).is_some()); } /// The probe is only worth carrying if it actually depends on the seed. diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index d924cc978..1bff9a72e 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -712,8 +712,8 @@ impl Lowering { }) } - /// The M0 fallback: one fragment holding everything, evaluated without - /// rewinding so top-level declarations still reach the caller. + /// The last-resort fallback: one fragment holding everything, evaluated + /// without rewinding so top-level declarations still reach the caller. fn whole_program_residual(&mut self, statements: &[Stmt]) { let body = wrap_statements(statements.to_vec()); let residual = self.push_residual(body); @@ -1185,6 +1185,9 @@ impl Lowering { Expr::IntegerConstant(value, ..) => self.constant(Dynamic::from(*value)), Expr::CharConstant(value, ..) => self.constant(Dynamic::from(*value)), Expr::StringConstant(value, ..) => self.constant(Dynamic::from(value.clone())), + // Rhai has no float literal to parse under `no_float`, so there is + // no variant to match. + #[cfg(not(feature = "no_float"))] Expr::FloatConstant(value, ..) => self.constant(Dynamic::from(**value)), // Folded by the optimizer, so it can hold anything a constant call // returned — including a function pointer, which must not be diff --git a/src/grain/compile/poolable.rs b/src/grain/compile/poolable.rs index 31f89ff99..4e7845809 100644 --- a/src/grain/compile/poolable.rs +++ b/src/grain/compile/poolable.rs @@ -17,14 +17,14 @@ use rhai::{Array, Blob, Dynamic, Map, INT}; /// created against. Keeping it out of the pool leaves it as a fragment, which /// evaluates through the path that does the attaching. pub(crate) fn is_poolable(value: &Dynamic) -> bool { - // `is_float` follows rhai's own `no_float`; the feature matrix for the - // device build is M6's problem, not this function's. - if value.is_unit() - || value.is_bool() - || value.is_int() - || value.is_char() - || value.is_string() - || value.is_float() + // Under `no_float` rhai has no float type and no `is_float` to ask, so + // there is nothing here for the question to be about. + #[cfg(not(feature = "no_float"))] + if value.is_float() { + return true; + } + + if value.is_unit() || value.is_bool() || value.is_int() || value.is_char() || value.is_string() { return true; } diff --git a/src/grain/format/abi.rs b/src/grain/format/abi.rs index 7203e97ad..1cdc00f4a 100644 --- a/src/grain/format/abi.rs +++ b/src/grain/format/abi.rs @@ -196,16 +196,19 @@ mod tests { #[test] fn a_narrower_int_is_refused_by_name() { + let host = Abi::host(); + // Halved rather than named: `only_i32` makes 4 the host's own width, + // and an artifact agreeing with the host is not a mismatch to report. let narrow = Abi { - int_bytes: 4, - ..Abi::host() + int_bytes: host.int_bytes / 2, + ..host }; assert_eq!( - narrow.incompatible_with(Abi::host()), + narrow.incompatible_with(host), Some(AbiMismatch::Width { what: "INT", - artifact: 4, - host: 8, + artifact: host.int_bytes / 2, + host: host.int_bytes, }), ); } diff --git a/src/grain/format/write.rs b/src/grain/format/write.rs index 06a2c1ecb..713cda0ca 100644 --- a/src/grain/format/write.rs +++ b/src/grain/format/write.rs @@ -29,8 +29,8 @@ pub enum WriteError { /// Where it is in the source pos: rhai::Position, }, - /// Script functions are still ASTs in the program's library. Compiling them - /// to chunks is M4. + /// The program still carries rhai's own function library rather than + /// chunks, so its functions are ASTs an artifact cannot hold. HasScriptFunctions, /// A pooled constant carries something that has no meaning in another /// process — a host type, a function pointer, a clock reading. diff --git a/src/grain/mod.rs b/src/grain/mod.rs index 5cee4eeac..ae19fe6c8 100644 --- a/src/grain/mod.rs +++ b/src/grain/mod.rs @@ -7,7 +7,8 @@ //! rhaigrain compiles the tree to a flat instruction stream that can be //! produced elsewhere and loaded without a parser. //! -//! `tests/allocation.rs` measures both ends of that with a tracking allocator. +//! `tests/grain/allocation.rs` measures both ends of that with a tracking +//! allocator. //! //! Execution reuses the host `Engine`: `Dynamic` stays the value type and every //! registered function is dispatched by rhai itself. Only control flow, local @@ -23,6 +24,63 @@ //! back to rhai's walker through [`bytecode::Op::EvalAst`], so a `Program` //! always means the same thing as the `AST` it came from. Progress is measured //! by [`Program::residual_count`] falling, not by constructs becoming legal. +//! +//! # Compiling and running +//! +//! The `Engine` does the parsing and, at runtime, all the dispatching; the VM +//! only replaces the walk between those two. +//! +//! ``` +//! use rhai::grain::{Compiler, Vm}; +//! use rhai::{Engine, Scope}; +//! +//! let engine = Engine::new(); +//! let ast = engine.compile("let total = 0; for i in 0..10 { total += i; } total")?; +//! +//! let program = Compiler::new().compile(&ast); +//! +//! // The `Scope` is the caller's: locals a script declares are left in it, +//! // exactly as `Engine::eval_with_scope` would. +//! let mut scope = Scope::new(); +//! let value = Vm::new(&engine).run(&program, &mut scope)?; +//! +//! assert_eq!(value.as_int().unwrap(), 45); +//! # Ok::<_, Box>(()) +//! ``` +//! +//! # Shipping an artifact +//! +//! The point of the byte encoding: compile on a host, run somewhere that never +//! sees the source. A loaded `Program` borrows its instructions from the bytes, +//! so nothing it retains grows with how long the script is. +//! +//! ``` +//! use rhai::grain::{Compiler, Program, Vm}; +//! use rhai::{Engine, Scope}; +//! +//! let engine = Engine::new(); +//! +//! // On the host. +//! let ast = engine.compile("let x = 6; x * 7")?; +//! let program = Compiler::new().compile(&ast); +//! +//! // `write` refuses a program still holding AST fragments, so this is also +//! // the check that the script lowered all the way through. +//! assert_eq!(program.residual_count(), 0); +//! let bytes = program.write().expect("no residuals, so it is writable"); +//! +//! // On the device, with no parser and no `AST` in sight. +//! let loaded = Program::read(&bytes).expect("written by this build"); +//! let value = Vm::new(&engine).run(&loaded, &mut Scope::new())?; +//! +//! assert_eq!(value.as_int().unwrap(), 42); +//! # Ok::<_, Box>(()) +//! ``` +//! +//! Diagnostics are separable: [`Program::write_stripped`] hands back the +//! artifact and its position table separately, so the device carries only the +//! first and a failure there comes back as an instruction address the host +//! turns into a source position with [`pos::resolve`]. // A VM that runs untrusted bytecode has no business containing any, and saying // so here makes it the compiler's problem rather than a promise. `crates/ diff --git a/tests/grain/allocation.rs b/tests/grain/allocation.rs index 5d2089d81..304fe722c 100644 --- a/tests/grain/allocation.rs +++ b/tests/grain/allocation.rs @@ -21,7 +21,7 @@ //! Host figures. They do not transfer to a device by halving: traffic-light //! measured the engine ratio at 0.71 rather than 0.5, and found the AST does //! not shrink on 32-bit at all, because `Dynamic`, `i64` and `f32` fields are -//! the same width either way. Device numbers come from the device, in M6. +//! the same width either way. Device numbers have to come from a device. use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicIsize, Ordering}; @@ -119,10 +119,12 @@ fn measure(f: impl FnOnce() -> T) -> Measured { } } -/// Stands in for a real control script until M1 wires in traffic-light's -/// `scripts/follow.rhai`, which is what the 24 bytes/source-byte figure was -/// measured against. Shaped like one: helper functions, integer and float -/// arithmetic, arrays, and a loop driving them. +/// A synthetic control, shaped like a real script: helper functions, integer +/// and float arithmetic, arrays, and a loop driving them. +/// +/// `FOLLOW` is the real one, and the figures that matter are measured against +/// it. This is here so the per-source-byte and peak/retained ratios have a +/// second shape to be read against. const SCRIPT: &str = r#" fn clamp(v, lo, hi) { if v < lo { lo } else if v > hi { hi } else { v } @@ -200,10 +202,10 @@ fn allocation_footprint() { println!("{:<24} {:>10} {:>10} {:>10}", "engine.compile (AST)", ast.bytes, ast.count, ast.peak); println!("{:<24} {:>10} {:>10} {:>10}", "Compiler::compile", program.bytes, program.count, program.peak); - // Not yet comparable to traffic-light's 24.0: that is a 32-bit device - // figure measured against minified source, this is a host figure against - // unminified source. M1 makes them comparable by measuring follow.rhai, - // minified, the way the server actually ships it. + // Not comparable to traffic-light's 24.0: that is a 32-bit device figure + // measured against minified source, this is a host figure against + // unminified source. Comparing them needs follow.rhai measured minified, + // the way the server actually ships it. println!("\nAST bytes per source byte {per_source_byte:.1} (host, unminified)"); println!("AST parser peak / retained {:.2}x (the peak is what has to fit)", ast.peak as f64 / ast.bytes as f64); @@ -213,14 +215,14 @@ fn allocation_footprint() { println!(" {label:>6} {n:>8}"); } - // The number M1 exists to produce: what the real script's tree costs, to - // set against what tests/projection.rs says a lowering of it would weigh. + // What the real script's tree costs, to set against what + // tests/grain/projection.rs says a lowering of it would weigh. // // Larger than traffic-light's own 77968 for the same script because this // builds rhai with default features. Theirs sets `no_module`, which drops // a `Namespace` (an inline `StaticVec` plus a hash) from every - // `Expr::Variable` payload — and this script has 457 of them. Compare - // against 77968, not against this, until M6 builds the restricted set. + // `Expr::Variable` payload — and this script has 457 of them. A restricted + // build is what 77968 should be compared against, not this one. println!("\nfollow.rhai — {} source bytes", FOLLOW.len()); println!("{:<24} {:>10} {:>10} {:>10}", "engine.compile (AST)", follow_ast.bytes, follow_ast.count, follow_ast.peak); println!("{:<24} {:>10.1}", "bytes per source byte", follow_ast.bytes as f64 / FOLLOW.len() as f64); @@ -295,15 +297,15 @@ fn allocation_footprint() { // scaling with length. assert!(ratios[2] > ratios[1] && ratios[1] > ratios[0], "the saving must grow with the program, got {ratios:?}",); - // The instrument has to be working before any milestone can lean on it. + // The instrument has to be working before anything can lean on it. assert!(ast.count > 0 && ast.bytes > 0, "the counters saw nothing; the global allocator is not installed",); assert!(ast.peak >= ast.bytes, "peak ({}) below retained ({}) means peak tracking is broken", ast.peak, ast.bytes,); - // M0's compiler clones the whole statement list into one residual, so the - // Program is a second copy of the tree rather than a saving. Pinning that - // here means M1's first real lowering shows up as this assertion failing, - // which is the intended way to notice progress. - assert!(program.bytes > 0, "a whole-program residual should cost about what the tree costs",); + // A lowered program retains its pools rather than a node per node, so it + // costs a fraction of the tree it came from. A `Program` that cost about + // what the tree cost would mean the script fell back to fragments, which + // hold real `Expr` trees — the one case where compiling saves nothing. + assert!(program.bytes > 0 && program.bytes < ast.bytes, "a lowered program must retain less than its tree, got {} against {}", program.bytes, ast.bytes,); drop(program); drop(ast); diff --git a/tests/grain/callback.rs b/tests/grain/callback.rs index 8b343d622..ee5f61dbe 100644 --- a/tests/grain/callback.rs +++ b/tests/grain/callback.rs @@ -109,7 +109,7 @@ fn a_capturing_closure_reaches_a_native_with_its_arguments_rotated() { #[test] fn stock_rhai_does_the_same_to_its_own_native_pointers() { let mut engine = corpus::engine(); - engine.register_fn("nsub", |a: i64, b: i64| a - b); + engine.register_fn("nsub", |a: rhai::INT, b: rhai::INT| a - b); let curried = "let s = \"ns\" + \"ub\"; let f = Fn(s).curry(10);"; assert_eq!(walk(&engine, &format!("{curried} [1, 2, 3].map(f)")), Ok("[-9, -8, -7]".to_string()), "rhai puts the element before the curried value",); diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index 953386321..66fef3802 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -10,6 +10,8 @@ //! runtime-state setup (function library, module resolver, source name, //! `return`/`exit` mapping) which is real code that can be wrong today. +use rhai::INT; + // Only `tests/fuzz.rs` and the `generated` fuzz target use this; the other // harnesses take the module for its cases. #[allow(dead_code)] @@ -29,10 +31,13 @@ pub struct Case { /// and the setter is the only way home — and rhai decides whether to call it /// from `func.is_method()`, which the VM cannot see and therefore approximates. /// Without a host type in the engine, nothing here is ever exercised. +/// Held as `INT` rather than `i64` throughout: `only_i32` narrows the script +/// integer, and a host type registered against the wider one would take a type +/// no script under that build can produce. #[derive(Debug, Clone, Default)] pub struct Widget { - pub level: i64, - pub cells: Vec, + pub level: INT, + pub cells: Vec, } #[derive(Debug, Clone, Default)] @@ -40,7 +45,7 @@ pub struct Holder { pub inner: Widget, } -fn out_of_range(index: i64, len: usize) -> Box { +fn out_of_range(index: INT, len: usize) -> Box { Box::new(rhai::EvalAltResult::ErrorArrayBounds(len, index, rhai::Position::NONE)) } @@ -50,13 +55,13 @@ pub fn engine() -> rhai::Engine { engine .register_type_with_name::("Widget") - .register_fn("widget", |level: i64| Widget { level, cells: vec![10, 20, 30] }) - .register_get_set("level", |w: &mut Widget| w.level, |w: &mut Widget, v: i64| w.level = v) + .register_fn("widget", |level: INT| Widget { level, cells: vec![10, 20, 30] }) + .register_get_set("level", |w: &mut Widget| w.level, |w: &mut Widget, v: INT| w.level = v) // Returning an error rather than panicking, because a panic in a // registered function takes the test process with it. .register_indexer_get_set( - |w: &mut Widget, i: i64| -> Result> { w.cells.get(i as usize).copied().ok_or_else(|| out_of_range(i, w.cells.len())) }, - |w: &mut Widget, i: i64, v: i64| -> Result<(), Box> { + |w: &mut Widget, i: INT| -> Result> { w.cells.get(i as usize).copied().ok_or_else(|| out_of_range(i, w.cells.len())) }, + |w: &mut Widget, i: INT, v: INT| -> Result<(), Box> { let len = w.cells.len(); *w.cells.get_mut(i as usize).ok_or_else(|| out_of_range(i, len))? = v; Ok(()) @@ -71,7 +76,7 @@ pub fn engine() -> rhai::Engine { engine .register_type_with_name::("Holder") - .register_fn("holder", |level: i64| Holder { inner: Widget { level, cells: vec![1, 2, 3] } }) + .register_fn("holder", |level: INT| Holder { inner: Widget { level, cells: vec![1, 2, 3] } }) .register_get_set("inner", |h: &mut Holder| h.inner.clone(), |h: &mut Holder, w: Widget| h.inner = w); engine @@ -416,7 +421,7 @@ pub const CASES: &[Case] = &[ case("error_wrong_arity", "fn f(a, b) { a } f(1)"), case("error_array_bounds", "let a = [1, 2]; a[10]"), // Rhai maps the *expected* type through its registered names and leaves the - // *actual* one raw, so a range guard reports `core::ops::range::Range` + // *actual* one raw, so a range guard reports `core::ops::range::Range` // rather than the `range` the same engine prints everywhere else. Mapping // both is the obvious mistake, and only a type with a registered name shows // it up. diff --git a/tests/grain/differential.rs b/tests/grain/differential.rs index ebcb46010..51ef22aaa 100644 --- a/tests/grain/differential.rs +++ b/tests/grain/differential.rs @@ -224,7 +224,7 @@ fn every_compiled_chunk_declares_the_stack_it_uses() { /// Residuals are the work left to do, so the count is the progress metric. /// -/// Prints the whole census so a milestone's effect is visible, and pins the +/// Prints the whole census so a change in coverage is visible, and pins the /// cases that should already be at zero. #[test] fn residual_census() { diff --git a/tests/grain/format.rs b/tests/grain/format.rs index 30656bdea..31c3770a1 100644 --- a/tests/grain/format.rs +++ b/tests/grain/format.rs @@ -12,7 +12,7 @@ use super::corpus; use rhai::grain::format::{ReadError, WriteError}; use rhai::grain::{Compiler, Program, Vm}; -use rhai::{Dynamic, Engine, Scope}; +use rhai::{Dynamic, Engine, Scope, INT}; /// What a run produced, in a form two runs can be compared on. #[derive(Debug, PartialEq, Eq)] @@ -130,7 +130,7 @@ const GOLDEN_ARTIFACT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/grain/ /// with it rather than being invented at each use. fn golden_scope() -> Scope<'static> { let mut scope = Scope::new(); - scope.push("supplied", vec![Dynamic::from(7_i64)]); + scope.push("supplied", vec![Dynamic::from(7 as INT)]); scope } @@ -270,7 +270,7 @@ fn a_chain_rooted_at_a_name_survives_the_round_trip() { let reloaded = Program::read(&bytes).expect("what we wrote must read back"); let seed = |scope: &mut Scope| { - scope.push("host", vec![Dynamic::from(1_i64)]); + scope.push("host", vec![Dynamic::from(1 as INT)]); }; let mut walked = Scope::new(); @@ -405,9 +405,12 @@ fn a_different_value_representation_is_refused_by_name() { let engine = corpus::engine(); let mut narrow = sample(&engine); - narrow[6] = 4; // INT width + // Halved rather than named: `only_i32` already makes 4 the host's own + // width, and an artifact agreeing with the host is not refused at all. + let half = narrow[6] / 2; // INT width + narrow[6] = half; let message = Program::read(&narrow).unwrap_err().to_string(); - assert!(message.contains("INT") && message.contains('4'), "the message must name the width: {message}",); + assert!(message.contains("INT") && message.contains(&half.to_string()), "the message must name the width: {message}",); let mut restricted = sample(&engine); restricted[8] ^= 0b100; // the `no_index` bit @@ -610,9 +613,9 @@ fn stripping_positions_shrinks_the_artifact() { /// The number this project exists to move: bytes retained per source byte, /// against the 24 a rhai `AST` costs on device. /// -/// This is the host-side artifact size, not device heap — M6 measures that. -/// What it establishes here is the encoding's own density, which is the part -/// the format controls. +/// This is the host-side artifact size, not device heap, which only a device +/// can report. What it establishes is the encoding's own density, which is the +/// part the format controls. #[test] fn artifact_size_census() { let engine = corpus::engine(); diff --git a/tests/grain/projection.rs b/tests/grain/projection.rs index 69f99244f..9ad8166ed 100644 --- a/tests/grain/projection.rs +++ b/tests/grain/projection.rs @@ -1,11 +1,11 @@ //! What would a lowering of this script actually weigh? //! -//! M1's job is to test the premise before building an instruction set to serve -//! it. `follow.rhai` turns out to use most of the language — nested index -//! writes, property chains, method calls, script functions, `loop`/`while`/ -//! `if-else` — so lowering it for real is several milestones of work. Doing -//! that first and *then* discovering the artifact is only marginally smaller -//! than the tree would be the expensive way to learn it. +//! This tests the premise before an instruction set is built to serve it. +//! `follow.rhai` uses most of the language — nested index writes, property +//! chains, method calls, script functions, `loop`/`while`/`if-else` — so +//! lowering it for real is a lot of work. Doing that first and *then* +//! discovering the artifact is only marginally smaller than the tree would be +//! the expensive way to learn it. //! //! So this counts the real AST, node by node, and prices each node against the //! planned stack encoding. The per-node costs are stated in `encoded_size` @@ -14,7 +14,8 @@ //! **This is a projection, not a measurement.** It is a lower bound in one //! direction and optimistic in another: it assumes every node lowers to the //! ops listed, and it ignores the operand stack traffic a real lowering emits -//! for temporaries. The measured number replaces it when M2-M4 land. +//! for temporaries. `tests/grain/format.rs` measures the artifact that +//! actually results, and that number is the one to trust. use std::collections::BTreeSet; diff --git a/tests/grain/scope.rs b/tests/grain/scope.rs index bdd26737e..9872acafd 100644 --- a/tests/grain/scope.rs +++ b/tests/grain/scope.rs @@ -20,7 +20,7 @@ use super::corpus; use rhai::grain::{Compiler, Vm}; -use rhai::{Dynamic, Engine, Module, Scope}; +use rhai::{Dynamic, Engine, Module, Scope, INT}; /// What a run produced, in a form two runs can be compared on. #[derive(Debug, PartialEq, Eq)] @@ -75,7 +75,9 @@ fn agree_with(engine: &Engine, source: &str, build: impl Fn(&mut Scope), writabl assert_eq!(actual, expected, "{source:?}"); } -fn lit(value: i64) -> Dynamic { +/// A script integer. Spelled through `INT` because `only_i32` narrows it, and +/// a caller variable holding the wider type is a host value no operator takes. +fn lit(value: INT) -> Dynamic { Dynamic::from(value) } @@ -84,7 +86,7 @@ fn a_caller_variable_can_be_read() { agree( "brightness * 2", |s| { - s.push("brightness", 21_i64); + s.push("brightness", 21 as INT); }, true, ); @@ -102,14 +104,14 @@ fn a_caller_variable_can_be_written() { agree( "brightness = 7; brightness", |s| { - s.push("brightness", 1_i64); + s.push("brightness", 1 as INT); }, true, ); agree( "brightness += 5; brightness", |s| { - s.push("brightness", 1_i64); + s.push("brightness", 1 as INT); }, true, ); @@ -135,7 +137,7 @@ fn a_caller_variable_in_first_argument_position_is_taken_by_reference() { agree( "push(log, 2); log", |s| { - s.push("log", vec![Dynamic::from(1_i64)]); + s.push("log", vec![lit(1)]); }, true, ); @@ -151,7 +153,7 @@ fn a_caller_variable_in_first_argument_position_is_taken_by_reference() { agree( "push(log, 2); log", |s| { - s.push_constant("log", vec![Dynamic::from(1_i64)]); + s.push_constant("log", vec![lit(1)]); }, true, ); @@ -161,7 +163,7 @@ fn a_caller_variable_in_first_argument_position_is_taken_by_reference() { agree( "nosuch(gone, brightness)", |s| { - s.push("brightness", 1_i64); + s.push("brightness", 1 as INT); }, true, ); @@ -177,7 +179,7 @@ fn a_caller_variable_in_first_argument_position_is_taken_by_reference() { /// that, because each fails differently. #[test] fn a_chain_can_be_rooted_at_a_caller_variable() { - let array = || vec![Dynamic::from(1_i64)]; + let array = || vec![lit(1)]; // A writable entry: read, mutate through a method, and assign through. agree( @@ -265,13 +267,13 @@ fn a_chain_rooted_at_a_resolved_name_cannot_be_written_through() { let mut engine = corpus::engine(); engine.on_var(|name, _, _| { Ok(match name { - "injected" => Some(Dynamic::from(vec![Dynamic::from(7_i64)])), + "injected" => Some(Dynamic::from(vec![lit(7)])), _ => None, }) }); let mut module = rhai::Module::new(); - module.set_var("published", vec![Dynamic::from(5_i64)]); + module.set_var("published", vec![lit(5)]); engine.register_global_module(module.into()); for source in [ @@ -305,8 +307,8 @@ fn a_closure_can_capture_a_caller_variable() { let seed = |scope: &mut Scope| { // Two of them, and the interesting one is not last: the index bug is // invisible with a single entry. - scope.push("first", vec![Dynamic::from(7_i64)]); - scope.push("second", vec![Dynamic::from(1_i64)]); + scope.push("first", vec![lit(7)]); + scope.push("second", vec![lit(1)]); }; // The write happens after the closure is made, so a captured copy answers @@ -325,7 +327,7 @@ fn a_local_shadows_the_caller_without_disturbing_it() { agree( "let brightness = 1; brightness += 1; brightness", |s| { - s.push("brightness", 100_i64); + s.push("brightness", 100 as INT); }, true, ); @@ -334,7 +336,7 @@ fn a_local_shadows_the_caller_without_disturbing_it() { agree( "let first = brightness; let brightness = 1; [first, brightness]", |s| { - s.push("brightness", 100_i64); + s.push("brightness", 100 as INT); }, true, ); @@ -389,7 +391,7 @@ fn a_script_function_name_is_not_a_variable() { fn a_global_module_constant_resolves() { let mut engine = corpus::engine(); let mut module = Module::new(); - module.set_var("CHANNELS", 512_i64); + module.set_var("CHANNELS", 512 as INT); engine.register_global_module(module.into()); let ast = engine.compile("CHANNELS / 2").expect("must compile"); @@ -414,7 +416,7 @@ fn a_variable_resolver_is_consulted_first() { let mut engine = corpus::engine(); engine.on_var(|name, _, _| { Ok(match name { - "injected" => Some(Dynamic::from(99_i64)), + "injected" => Some(lit(99)), // Declining must fall through to the scope rather than fail. _ => None, }) @@ -428,7 +430,7 @@ fn a_variable_resolver_is_consulted_first() { }; let mut scope = Scope::new(); - scope.push("ordinary", 5_i64); + scope.push("ordinary", 5 as INT); let program = compile(&engine, "injected + ordinary"); let value = Vm::new(&engine).run(&program, &mut scope.clone()).expect("both must resolve"); @@ -453,8 +455,8 @@ fn a_resolver_that_grows_the_scope_forces_a_search() { let mut engine = corpus::engine(); engine.on_var(|name, _, mut context| { if name == "grow" { - context.scope_mut().push("added", 1_i64); - return Ok(Some(Dynamic::from(1_i64))); + context.scope_mut().push("added", 1 as INT); + return Ok(Some(lit(1))); } Ok(None) }); @@ -487,7 +489,7 @@ fn a_resolved_receiver_is_not_the_scope_entry_it_shadows() { let mut engine = corpus::engine(); engine.on_var(|name, _, _| { Ok(match name { - "shadowed" => Some(Dynamic::from(vec![Dynamic::from(9_i64)])), + "shadowed" => Some(Dynamic::from(vec![lit(9)])), _ => None, }) }); @@ -498,7 +500,7 @@ fn a_resolved_receiver_is_not_the_scope_entry_it_shadows() { assert_eq!(program.residual_count(), 0, "the call must lower"); let start = |scope: &mut Scope| { - scope.push("shadowed", vec![Dynamic::from(1_i64)]); + scope.push("shadowed", vec![lit(1)]); }; let mut walked = Scope::new(); @@ -557,13 +559,13 @@ fn a_compiled_function_can_be_called_by_name() { let mut vm = Vm::new(&engine); let value = vm - .call_function(&program, "add", vec![Dynamic::from(2_i64), Dynamic::from(3_i64)], 0, rhai::Position::NONE) + .call_function(&program, "add", vec![lit(2), lit(3)], 0, rhai::Position::NONE) .expect("must call"); assert_eq!(value.as_int().unwrap(), 5); // Wrong arity is a miss, not a crash — the table is keyed on both. let err = vm - .call_function(&program, "add", vec![Dynamic::from(1_i64)], 0, rhai::Position::NONE) + .call_function(&program, "add", vec![lit(1)], 0, rhai::Position::NONE) .expect_err("one argument is a different function"); assert!(matches!(*err, rhai::EvalAltResult::ErrorFunctionNotFound(..))); @@ -573,7 +575,7 @@ fn a_compiled_function_can_be_called_by_name() { // The operand stack is where it started, so a caller can keep using it. let value = vm - .call_function(&program, "add", vec![Dynamic::from(10_i64), Dynamic::from(1_i64)], 0, rhai::Position::NONE) + .call_function(&program, "add", vec![lit(10), lit(1)], 0, rhai::Position::NONE) .expect("must call again"); assert_eq!(value.as_int().unwrap(), 11); } @@ -625,7 +627,7 @@ fn a_program_reading_caller_state_can_be_written() { let reloaded = rhai::grain::Program::read(&bytes).expect("must load"); let mut scope = Scope::new(); - scope.push("brightness", 5_i64); + scope.push("brightness", 5 as INT); scope.push("mode", "chase".to_string()); let value = Vm::new(&engine).run(&reloaded, &mut scope).expect("must run"); From c2644b66550b861635c5a48418f4afb812ede633 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 20:15:38 -0700 Subject: [PATCH 08/33] fmt --- tests/grain/scope.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/grain/scope.rs b/tests/grain/scope.rs index 9872acafd..6f6d84c1d 100644 --- a/tests/grain/scope.rs +++ b/tests/grain/scope.rs @@ -558,9 +558,7 @@ fn a_compiled_function_can_be_called_by_name() { let program = Compiler::new().compile(&ast); let mut vm = Vm::new(&engine); - let value = vm - .call_function(&program, "add", vec![lit(2), lit(3)], 0, rhai::Position::NONE) - .expect("must call"); + let value = vm.call_function(&program, "add", vec![lit(2), lit(3)], 0, rhai::Position::NONE).expect("must call"); assert_eq!(value.as_int().unwrap(), 5); // Wrong arity is a miss, not a crash — the table is keyed on both. @@ -574,9 +572,7 @@ fn a_compiled_function_can_be_called_by_name() { assert!(matches!(*err, rhai::EvalAltResult::ErrorInFunctionCall(..)), "got {err:?}",); // The operand stack is where it started, so a caller can keep using it. - let value = vm - .call_function(&program, "add", vec![lit(10), lit(1)], 0, rhai::Position::NONE) - .expect("must call again"); + let value = vm.call_function(&program, "add", vec![lit(10), lit(1)], 0, rhai::Position::NONE).expect("must call again"); assert_eq!(value.as_int().unwrap(), 11); } From 658f43f2f867d925d9ded4169a8bf314b74de337 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 20:38:11 -0700 Subject: [PATCH 09/33] remove std references --- src/grain/bytecode/strings.rs | 2 +- src/grain/vm/callback.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/grain/bytecode/strings.rs b/src/grain/bytecode/strings.rs index 169346c60..b5159c240 100644 --- a/src/grain/bytecode/strings.rs +++ b/src/grain/bytecode/strings.rs @@ -1,4 +1,4 @@ -use std::borrow::Cow; +use alloc::borrow::Cow; /// Every name a program mentions, as one blob and a list of spans. /// diff --git a/src/grain/vm/callback.rs b/src/grain/vm/callback.rs index d718011be..929a7901d 100644 --- a/src/grain/vm/callback.rs +++ b/src/grain/vm/callback.rs @@ -34,8 +34,8 @@ //! Neither touches a pointer called directly from compiled code, which is //! `Op::CallFnPtr` and never comes through here. -use std::any::TypeId; -use std::mem; +use core::any::TypeId; +use core::mem; use crate::{ func::RhaiFunc, Dynamic, FuncRegistration, ImmutableString, Module, NativeCallContext, Shared, From ad4b3a5b1ba3ad915cef5855c27a4ae2c807d4a3 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 20:39:17 -0700 Subject: [PATCH 10/33] remove MORE std references --- src/grain/bytecode/verify.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/grain/bytecode/verify.rs b/src/grain/bytecode/verify.rs index ed2bb5fa2..ae391fd69 100644 --- a/src/grain/bytecode/verify.rs +++ b/src/grain/bytecode/verify.rs @@ -661,7 +661,7 @@ mod tests { let chunk = Chunk::new(0, code.len() as u32, 8); let pools = Pools { names: 1, - chains: std::slice::from_ref(&chain), + chains: core::slice::from_ref(&chain), ..pools() }; assert!( @@ -691,7 +691,7 @@ mod tests { &[chunk], Pools { names: 1, - chains: std::slice::from_ref(&assigning), + chains: core::slice::from_ref(&assigning), ..pools() }, ), From a7747b3018edcb4dbe0ef583a01e2673a6022697 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 20:40:35 -0700 Subject: [PATCH 11/33] typo --- src/grain/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/grain/mod.rs b/src/grain/mod.rs index ae19fe6c8..baa496076 100644 --- a/src/grain/mod.rs +++ b/src/grain/mod.rs @@ -39,7 +39,7 @@ //! //! let program = Compiler::new().compile(&ast); //! -//! // The `Scope` is the caller's: locals a script declares are left in it, +//! // The `Scope` is the caller's locals: a script declares are left in it, //! // exactly as `Engine::eval_with_scope` would. //! let mut scope = Scope::new(); //! let value = Vm::new(&engine).run(&program, &mut scope)?; From 4c6a79b4585ec7fcd13e0bacc0f939dd2ec5e3a1 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 20:51:55 -0700 Subject: [PATCH 12/33] ensure no_std feature works, add CI --- .github/workflows/build.yml | 1 + src/grain/bytecode/chain.rs | 3 +++ src/grain/bytecode/code.rs | 2 ++ src/grain/bytecode/positions.rs | 2 ++ src/grain/bytecode/strings.rs | 2 ++ src/grain/bytecode/switch.rs | 2 ++ src/grain/bytecode/verify.rs | 2 ++ src/grain/compile/cases.rs | 2 ++ src/grain/compile/mod.rs | 2 ++ src/grain/compile/slots.rs | 2 ++ src/grain/format/mod.rs | 2 ++ src/grain/format/read.rs | 2 ++ src/grain/format/write.rs | 2 ++ src/grain/program.rs | 3 +++ src/grain/vm/callback.rs | 2 ++ src/grain/vm/mod.rs | 2 ++ 16 files changed, 33 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc6f59316..c40694e25 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,6 +109,7 @@ jobs: - {os: ubuntu-latest, flags: "--profile unix", experimental: false} - {os: windows-latest, flags: "--profile windows", experimental: true} - {os: macos-latest, flags: "--profile macos", experimental: false} + - {os: ubuntu-latest, flags: "--profile unix --features rhai/grain", experimental: false} steps: - name: Checkout uses: actions/checkout@v4 diff --git a/src/grain/bytecode/chain.rs b/src/grain/bytecode/chain.rs index 221e8dcb9..aae026d72 100644 --- a/src/grain/bytecode/chain.rs +++ b/src/grain/bytecode/chain.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + /// One step along `a.b[i].c(x)`. /// /// Steps live in the program's chain pool rather than in the instruction diff --git a/src/grain/bytecode/code.rs b/src/grain/bytecode/code.rs index 8e814af9c..98e30e68a 100644 --- a/src/grain/bytecode/code.rs +++ b/src/grain/bytecode/code.rs @@ -26,6 +26,8 @@ use alloc::borrow::Cow; use crate::grain::bytecode::{Op, Receiver}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; /// Instruction tags. /// diff --git a/src/grain/bytecode/positions.rs b/src/grain/bytecode/positions.rs index 56ca95693..6a3c4e533 100644 --- a/src/grain/bytecode/positions.rs +++ b/src/grain/bytecode/positions.rs @@ -1,5 +1,7 @@ use crate::grain::pos::Site; use crate::Position; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; /// Where each instruction came from, or nothing. /// diff --git a/src/grain/bytecode/strings.rs b/src/grain/bytecode/strings.rs index b5159c240..01a4bc98e 100644 --- a/src/grain/bytecode/strings.rs +++ b/src/grain/bytecode/strings.rs @@ -1,4 +1,6 @@ use alloc::borrow::Cow; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; /// Every name a program mentions, as one blob and a list of spans. /// diff --git a/src/grain/bytecode/switch.rs b/src/grain/bytecode/switch.rs index 182e81d46..3e069c01f 100644 --- a/src/grain/bytecode/switch.rs +++ b/src/grain/bytecode/switch.rs @@ -1,4 +1,6 @@ use crate::{ast::RangeCase, Dynamic, INT}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; /// A `switch`, as one dispatch table. /// diff --git a/src/grain/bytecode/verify.rs b/src/grain/bytecode/verify.rs index ae391fd69..481d4862a 100644 --- a/src/grain/bytecode/verify.rs +++ b/src/grain/bytecode/verify.rs @@ -1,5 +1,7 @@ use crate::grain::bytecode::code::{self, tag}; use crate::grain::bytecode::{Chain, Chunk, Op, Receiver, Root, Step, Switch, Tail}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; /// What the pools hold, so an instruction's indices can be checked against /// something. diff --git a/src/grain/compile/cases.rs b/src/grain/compile/cases.rs index b2dc2eb9d..22732a718 100644 --- a/src/grain/compile/cases.rs +++ b/src/grain/compile/cases.rs @@ -13,6 +13,8 @@ //! sharing a case value. use crate::{ast::RangeCase, INT}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; use crate::grain::bytecode::SwitchRange; diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index 1bff9a72e..3fe461222 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -3,6 +3,8 @@ mod poolable; mod slots; use core::mem; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; use crate::ast::{ ASTFlags, ASTNode, Expr, FlowControl, FnCallExpr, OpAssignment, ScriptFuncDef, Stmt, StmtBlock, diff --git a/src/grain/compile/slots.rs b/src/grain/compile/slots.rs index 577508a95..3dc8fe7f0 100644 --- a/src/grain/compile/slots.rs +++ b/src/grain/compile/slots.rs @@ -1,4 +1,6 @@ use rhai::ImmutableString; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; /// Assigns a slot to every local, mirroring how rhai's own `Scope` grows. /// diff --git a/src/grain/format/mod.rs b/src/grain/format/mod.rs index 094ed0efa..8572c1107 100644 --- a/src/grain/format/mod.rs +++ b/src/grain/format/mod.rs @@ -39,6 +39,8 @@ //! not something a script author can act on. use core::convert::TryFrom; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; mod abi; mod read; diff --git a/src/grain/format/read.rs b/src/grain/format/read.rs index 5ec526a55..3f9452a8e 100644 --- a/src/grain/format/read.rs +++ b/src/grain/format/read.rs @@ -1,5 +1,7 @@ use crate::{tokenizer::Token, Dynamic, ImmutableString}; use core::convert::{TryFrom, TryInto}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; use crate::grain::bytecode::{ AssignOp, BadTable, Chain, Chunk, Positions, Root, Step, Strings, Switch, SwitchCase, diff --git a/src/grain/format/write.rs b/src/grain/format/write.rs index 713cda0ca..ed4d46987 100644 --- a/src/grain/format/write.rs +++ b/src/grain/format/write.rs @@ -1,4 +1,6 @@ use core::ops::{Range, RangeInclusive}; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; use rhai::{tokenizer::Token, Array, Blob, Dynamic, Map, INT}; diff --git a/src/grain/program.rs b/src/grain/program.rs index d936bbd58..3ffe07542 100644 --- a/src/grain/program.rs +++ b/src/grain/program.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "no_std")] +use std::prelude::v1::*; + use crate::ast::{ASTFlags, ASTNode}; #[cfg(not(feature = "no_module"))] use crate::module_resolvers::StaticModuleResolver; diff --git a/src/grain/vm/callback.rs b/src/grain/vm/callback.rs index 929a7901d..b02310dd2 100644 --- a/src/grain/vm/callback.rs +++ b/src/grain/vm/callback.rs @@ -36,6 +36,8 @@ use core::any::TypeId; use core::mem; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; use crate::{ func::RhaiFunc, Dynamic, FuncRegistration, ImmutableString, Module, NativeCallContext, Shared, diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index b5d9ab3fc..5d0d788d9 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -1,4 +1,6 @@ use core::mem; +#[cfg(feature = "no_std")] +use std::prelude::v1::*; use crate::engine::{FN_IDX_GET, FN_IDX_SET}; use crate::eval::calc_data_sizes; From 80529a41c9eda0a50b7ed271ad510a36418515bf Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Fri, 7 Aug 2026 21:06:43 -0700 Subject: [PATCH 13/33] fail on import --- src/grain/compile/mod.rs | 11 +++++++++++ tests/grain/scope.rs | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index 3fe461222..1e322afc6 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -1164,6 +1164,17 @@ impl Lowering { true } + // The one statement the fragment fallback below cannot hold. + // + // `import` declares into the imports stack rather than the scope, + // and a fragment that rewinds truncates that stack on the way out + // (`eval/stmt.rs:55`) — so the alias would be gone before the next + // statement could name it, and a qualified call is its own + // fragment. Refusing the lowering hands the body to the walker + // whole, which is where the alias lives long enough to be used. + #[cfg(not(feature = "no_module"))] + Stmt::Import(..) => false, + // Not lowered yet. Safe as a fragment because every remaining // statement kind either declares nothing or rewinds what it // declares, so the scope is the same shape afterwards. diff --git a/tests/grain/scope.rs b/tests/grain/scope.rs index 6f6d84c1d..36af79e4f 100644 --- a/tests/grain/scope.rs +++ b/tests/grain/scope.rs @@ -408,6 +408,44 @@ fn a_global_module_constant_resolves() { assert!(matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), "got {err:?}",); } +/// A script `import` costs the whole body its lowering, and must. +/// +/// `import` declares into the imports stack, not the scope. A per-statement +/// fragment rewinds that stack on the way out (`eval/stmt.rs:55`), so the alias +/// would be dropped before the qualified call — itself a separate fragment — +/// could name it, and the VM answered `Module not found` where the walker +/// answered. That is a wrong result rather than a missing feature, which is the +/// one thing the fragment fallback is not allowed to produce. +/// +/// So the compiler refuses the lowering instead, and the walker takes the body +/// as one block. These cases are here to keep it refusing: an `import` that +/// started lowering again would put the divergence straight back. +#[test] +fn an_import_keeps_the_walkers_answer() { + let mut engine = corpus::engine(); + + let mut resolver = rhai::module_resolvers::StaticModuleResolver::new(); + let module_ast = engine.compile("fn double(x) { x * 2 } export const LIMIT = 99;").expect("the module source must parse"); + let module = Module::eval_ast_as_new(Scope::new(), &module_ast, &engine).expect("the module must build"); + resolver.insert("kit", module); + engine.set_module_resolver(resolver); + + // An `import` in the body costs the body its lowering, so the program is a + // fragment rhai's walker has to evaluate and cannot become an artifact. + for source in [ + r#"import "kit" as k; k::double(21)"#, + r#"import "kit" as k; k::LIMIT"#, + r#"let r = 0; { import "kit" as k; r = k::double(4); } r"#, + r#"import "kit" as k; let t = 0; for i in 0..3 { t += k::double(i); } t"#, + ] { + agree_with(&engine, source, |_| {}, false); + } + + // In a function body the fallback is per-function: that body stays an AST + // in the library and the top level still lowers whole. + agree_with(&engine, r#"fn via_kit() { import "kit" as k; k::double(3) } via_kit()"#, |_| {}, true); +} + /// The first of the three, and the one a VM would most plausibly skip: a /// resolver the host registered through `Engine::on_var` sees the name before /// the scope does. From b933a11af637c124cf8da95b285c3cf6577a6257 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 04:14:30 -0700 Subject: [PATCH 14/33] more edge cases, function names --- Cargo.toml | 6 + examples/grain_bench.rs | 261 +++++++++++++++++++++++++++++++++ fuzz/fuzz_targets/generated.rs | 4 +- fuzz/fuzz_targets/load.rs | 2 +- fuzz/fuzz_targets/roundtrip.rs | 8 +- src/grain/compile/mod.rs | 76 +++++++++- src/grain/format/mod.rs | 6 + src/grain/mod.rs | 4 +- src/grain/program.rs | 5 +- src/grain/vm/mod.rs | 193 ++++++++++++++++++++---- tests/grain/allocation.rs | 5 + tests/grain/callback.rs | 6 +- tests/grain/corpus/generate.rs | 4 + tests/grain/corpus/mod.rs | 32 ++++ tests/grain/differential.rs | 32 ++-- tests/grain/format.rs | 214 +++++++++++++++------------ tests/grain/fuzz.rs | 12 +- tests/grain/limits.rs | 8 +- tests/grain/scope.rs | 107 ++++++++++++-- 19 files changed, 805 insertions(+), 180 deletions(-) create mode 100644 examples/grain_bench.rs diff --git a/Cargo.toml b/Cargo.toml index c7708e476..082550a10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,6 +149,12 @@ required-features = ["serde"] name = "definitions" required-features = ["metadata", "internals"] +# VM-versus-walker timings. Run with `--release`; `-- --check` exits non-zero +# on a case that has fallen below the floor recorded beside it. +[[example]] +name = "grain_bench" +required-features = ["grain"] + # The grain harnesses, as one binary. `tests/mod.rs` wraps them in an inline # `mod grain`, which is what puts their module paths inside `tests/grain/`. [[test]] diff --git a/examples/grain_bench.rs b/examples/grain_bench.rs new file mode 100644 index 000000000..d204afb85 --- /dev/null +++ b/examples/grain_bench.rs @@ -0,0 +1,261 @@ +//! Rough VM-versus-walker timings, on the same AST. +//! +//! Indicative, not criterion: repeated runs, reporting the fastest of each and +//! the spread around it so a single scheduling hiccup does not read as a +//! result. Run with `--release`; a debug build measures bounds checks more than +//! anything else. +//! +//! # Catching a regression +//! +//! `cargo run --release --example bench -- --check` exits non-zero if any case +//! has fallen below the floor recorded beside it. +//! +//! What is compared is the **ratio**, not the time. Absolute milliseconds say +//! as much about the machine as about the code, and there is no useful way to +//! commit one; the walker and the VM run back to back on the same machine in +//! the same process, so their ratio mostly divides the machine out. Mostly, not +//! entirely — cache size and core count still move it — so a floor sits about +//! 15% under the observed figure. That is wide enough not to cry wolf on a +//! slower runner and tight enough that losing a fast path shows up. + +use std::time::{Duration, Instant}; + +use rhai::grain::{Compiler, Program, Vm}; +use rhai::{Dynamic, Engine, Scope}; + +const RUNS: usize = 9; + +struct Case { + name: &'static str, + source: &'static str, + iterations: usize, + /// Whether the run needs the callback wrappers installed, which costs an + /// owned program and a module built per run. + callbacks: bool, + /// The speedup this case must not drop below. + /// + /// Beside the source rather than in a table of its own, so changing one + /// without the other is visible in the diff. + floor: f64, +} + +/// The fastest sample, and how much slower the middle one was. +/// +/// Noise on a timing is one-sided — nothing makes a run finish sooner than it +/// can — so the fastest sample is the least contaminated estimate, and the +/// median is here only to say how contaminated the rest were. A wide spread +/// means the number below it should not be read closely. +struct Timing { + fastest: Duration, + median: Duration, +} + +impl Timing { + fn secs(&self) -> f64 { + self.fastest.as_secs_f64() + } + + /// How far the median sits above the fastest, as a fraction. + fn spread(&self) -> f64 { + self.median.as_secs_f64() / self.fastest.as_secs_f64() - 1.0 + } +} + +const CASES: &[Case] = &[ + Case { + name: "tight integer loop", + source: "let s = 0; let i = 0; while i < 20000 { s += i; i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.30, + }, + Case { + name: "float arithmetic", + source: "let x = 0.0; let i = 0; while i < 20000 { x += (i.to_float() * 1.5) / 2.5; i += 1; } x", + iterations: 20, + callbacks: false, + floor: 1.10, + }, + Case { + name: "script fn calls", + source: "fn add(a, b) { a + b } let s = 0; let i = 0; while i < 5000 { s = add(s, i); i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.55, + }, + // The VM scans its case hashes; rhai probes a hash map. Two sizes, + // because which of those wins is a question about how many arms there + // are, and a `switch` nobody would write is the only place the scan can + // lose. + Case { + name: "switch, 4 arms", + source: "let s = 0; let i = 0; while i < 20000 { \ + switch i % 4 { 0 => s += 1, 1 => s += 2, 2 => s += 3, _ => s += 4 } \ + i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.40, + }, + Case { + name: "switch, 16 arms", + source: "let s = 0; let i = 0; while i < 20000 { \ + switch i % 16 { \ + 0 => s += 1, 1 => s += 2, 2 => s += 3, 3 => s += 4, \ + 4 => s += 5, 5 => s += 6, 6 => s += 7, 7 => s += 8, \ + 8 => s += 9, 9 => s += 10, 10 => s += 11, 11 => s += 12, \ + 12 => s += 13, 13 => s += 14, 14 => s += 15, _ => s += 16 } \ + i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.35, + }, + Case { + name: "branch heavy", + source: "let s = 0; let i = 0; while i < 20000 { if i % 3 == 0 { s += 1; } else if i % 3 == 1 { s += 2; } else { s -= 1; } i += 1; } s", + iterations: 20, + callbacks: false, + floor: 1.45, + }, + // The one case the VM is expected to lose. Every element is a boundary out + // of the VM, through rhai's dispatch and back into a second `Vm` with an + // empty resolution cache — where the walker stays inside itself and reaches + // the closure body directly. 1000 crossings per iteration. + Case { + name: "native callbacks", + source: "let a = []; let i = 0; while i < 500 { a.push(i); i += 1; } \ + let b = a.map(|x| x * 2); b.filter(|x| x % 3 == 0).len()", + iterations: 20, + callbacks: true, + floor: 0.25, + }, +]; + +fn time(mut run: impl FnMut()) -> Timing { + let mut samples: Vec = (0..RUNS) + .map(|_| { + let start = Instant::now(); + run(); + start.elapsed() + }) + .collect(); + samples.sort_unstable(); + Timing { + fastest: samples[0], + median: samples[RUNS / 2], + } +} + +fn main() { + let check = std::env::args().any(|arg| arg == "--check"); + let engine = Engine::new(); + + // Rhai's default options include FAST_OPS, which makes the walker + // short-circuit binary operators and op-assignments straight to builtin + // function pointers — no hash, no resolution cache + // (`func/call.rs:1775-1799`, `eval/stmt.rs:131-148`). Turning it off + // measures how much of the walker's speed comes from that, and therefore + // how much of the VM's planned "typed fast opcodes" win is already taken. + let mut slow_engine = Engine::new(); + slow_engine.set_fast_operators(false); + + println!( + "{:<22} {:>11} {:>11} {:>9} {:>7} {:>8} {:>11} {:>10}", + "", "walker", "vm", "speedup", "floor", "spread", "walker-slow", "fragments" + ); + + let mut below_floor = Vec::new(); + + for case in CASES { + let ast = engine.compile(case.source).expect("must compile"); + let program: Program = Compiler::new().compile(&ast); + // Owned and shared only where a pointer can escape to a native, so the + // ordinary cases keep measuring the ordinary path. + let shared = case + .callbacks + .then(|| Compiler::new().compile(&ast).into_shared()); + let run_vm = || match &shared { + Some(shared) => Vm::new(&engine).eval_with_callbacks(&mut Scope::new(), shared), + None => Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program), + }; + + // Same result, or the comparison is meaningless. + let expected = engine + .eval_ast_with_scope::(&mut Scope::new(), &ast) + .expect("walker must succeed"); + let actual = run_vm().expect("vm must succeed"); + assert_eq!( + format!("{expected:?}"), + format!("{actual:?}"), + "{} disagreed, so its timing means nothing", + case.name, + ); + + let walker = time(|| { + for _ in 0..case.iterations { + let _ = engine + .eval_ast_with_scope::(&mut Scope::new(), &ast) + .unwrap(); + } + }); + + let vm = time(|| { + for _ in 0..case.iterations { + let _ = run_vm().unwrap(); + } + }); + + let slow_ast = slow_engine.compile(case.source).expect("must compile"); + let walker_slow = time(|| { + for _ in 0..case.iterations { + let _ = slow_engine + .eval_ast_with_scope::(&mut Scope::new(), &slow_ast) + .unwrap(); + } + }); + + // The spread reported is the VM's, because that is the number the + // floor is about. A walker sample knocked sideways shows up in the + // speedup anyway. + let speedup = walker.secs() / vm.secs(); + println!( + "{:<22} {:>9.1}ms {:>9.1}ms {:>8.2}x {:>6.2}x {:>7.0}% {:>9.1}ms {:>10}", + case.name, + walker.secs() * 1000.0, + vm.secs() * 1000.0, + speedup, + case.floor, + vm.spread() * 100.0, + walker_slow.secs() * 1000.0, + program.residual_nodes(), + ); + + if speedup < case.floor { + below_floor.push(format!( + "\n {}: {speedup:.2}x, floor {:.2}x (VM samples spread {:.0}%)", + case.name, + case.floor, + vm.spread() * 100.0, + )); + } + } + + if below_floor.is_empty() { + return; + } + + // Printed whether or not this is a gated run: a regression is worth seeing + // even when nobody asked for an exit code. + eprintln!( + "\n{} case(s) below their floor:{}", + below_floor.len(), + below_floor.join(""), + ); + eprintln!( + "\nA wide spread means the machine was busy — rerun before believing it. \ + If the loss is real, either find it or move the floor in the same commit \ + that causes it.", + ); + if check { + std::process::exit(1); + } +} diff --git a/fuzz/fuzz_targets/generated.rs b/fuzz/fuzz_targets/generated.rs index 9751290f2..a86b9aab3 100644 --- a/fuzz/fuzz_targets/generated.rs +++ b/fuzz/fuzz_targets/generated.rs @@ -113,9 +113,9 @@ fn compare(engine: &Engine, source: &str) -> Option<(String, String)> { let mut vm_scope = Scope::new(); let ours = if program.makes_fn_pointers() { let program = program.into_shared(); - Vm::new(engine).run_with_callbacks(&program, &mut vm_scope) + Vm::new(engine).eval_with_callbacks(&mut vm_scope, &program) } else { - Vm::new(engine).run(&program, &mut vm_scope) + Vm::new(engine).eval_with_scope(&mut vm_scope, &program) }; Some((outcome(&vm_scope, ours)?, expected)) } diff --git a/fuzz/fuzz_targets/load.rs b/fuzz/fuzz_targets/load.rs index 5ea94dc83..75d686f8d 100644 --- a/fuzz/fuzz_targets/load.rs +++ b/fuzz/fuzz_targets/load.rs @@ -42,5 +42,5 @@ fuzz_target!(|data: &[u8]| { engine.set_max_map_size(64); engine.set_max_call_levels(16); - let _ = Vm::new(&engine).run(&program, &mut Scope::new()); + let _ = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program); }); diff --git a/fuzz/fuzz_targets/roundtrip.rs b/fuzz/fuzz_targets/roundtrip.rs index 2d665d8a2..59400d99d 100644 --- a/fuzz/fuzz_targets/roundtrip.rs +++ b/fuzz/fuzz_targets/roundtrip.rs @@ -105,7 +105,7 @@ fn optimizer_lost_a_local(source: &str, direct: &str) -> bool { let program = Compiler::new().compile(&ast); let expected = outcome(|scope| plain.eval_ast_with_scope::(scope, &ast)); - let ours = outcome(|scope| Vm::new(&plain).run(&program, scope)); + let ours = outcome(|scope| Vm::new(&plain).eval_with_scope(scope, &program)); ours == expected } @@ -124,8 +124,8 @@ fuzz_target!(|source: String| { .makes_fn_pointers() .then(|| Compiler::new().compile(&ast).into_shared()); let run = |scope: &mut Scope| match &shared { - Some(shared) => Vm::new(&engine).run_with_callbacks(shared, scope), - None => Vm::new(&engine).run(&program, scope), + Some(shared) => Vm::new(&engine).eval_with_callbacks(scope, shared), + None => Vm::new(&engine).eval_with_scope(scope, &program), }; let direct = outcome(run); @@ -146,7 +146,7 @@ fuzz_target!(|source: String| { return; }; let reloaded = Program::read(&bytes).expect("what we wrote must read back"); - let loaded = outcome(|scope| Vm::new(&engine).run(&reloaded, scope)); + let loaded = outcome(|scope| Vm::new(&engine).eval_with_scope(scope, &reloaded)); assert_eq!( loaded, expected, diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index 1e322afc6..bc2f5d85f 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -769,6 +769,12 @@ impl Lowering { true } + // Standing alone is the position `eval` is usually written in, and + // rhai gives it its own node — so this is the arm that catches it, + // not the `Expr::FnCall` one. See there for why it defeats the + // lowering rather than becoming a fragment. + Stmt::FnCall(call, ..) if call.name == crate::engine::KEYWORD_EVAL => false, + // A plain local on the left. Stmt::Assignment(payload) if matches!(&payload.1.lhs, Expr::Variable(v, ..) @@ -1175,10 +1181,31 @@ impl Lowering { #[cfg(not(feature = "no_module"))] Stmt::Import(..) => false, - // Not lowered yet. Safe as a fragment because every remaining - // statement kind either declares nothing or rewinds what it - // declares, so the scope is the same shape afterwards. - other => { + // Not lowered yet, and listed rather than matched with `_` on + // purpose. A wildcard here silently turned `import` and `eval` + // into fragments that answered differently from the walker; naming + // every kind means a new one added to rhai's AST stops the build + // until someone has decided which of the three it is — lowered, + // fragment, or too scope-shaped to be either. + // + // The ones below are fragments because each either declares + // nothing or rewinds what it declares, so the scope is the same + // shape afterwards. That is the property to check before adding to + // this list. + other @ (Stmt::Noop(..) + | Stmt::FnCall(..) + | Stmt::Assignment(..) + | Stmt::Return(..)) => { + let residual = self.push_residual(wrap_statements(vec![other.clone()])); + self.emit(Op::EvalAst { + residual, + rewind_scope: true, + }); + true + } + + #[cfg(not(feature = "no_module"))] + other @ Stmt::Export(..) => { let residual = self.push_residual(wrap_statements(vec![other.clone()])); self.emit(Op::EvalAst { residual, @@ -1274,6 +1301,18 @@ impl Lowering { self.lower_call(call, *pos); } + // `eval` evaluates a script in the *caller's* scope, so what it + // declares outlives it and the next statement can name it. The + // slot model resolved its indices against a scope that does not + // have those entries, so a lowered read past an `eval` looks in + // the wrong place — `eval("let x = 40"); x + 2` found no `x` where + // the walker found 40. Refusing the lowering hands the body to the + // walker, which is the only thing that knows the real shape. + Expr::FnCall(call, ..) if call.name == crate::engine::KEYWORD_EVAL => { + self.residual_expr(expr); + self.defeated = true; + } + // A literal whose elements are all constant never reaches here — // rhai's optimizer folds it into a `DynamicConstant` first — so // this is the one that has to be built at run time. @@ -1395,7 +1434,34 @@ impl Lowering { } } - _ => self.residual_expr(expr), + // Custom syntax runs host code against an `EvalContext`, which can + // declare into the caller's scope. What it declares is invisible + // here, so the slot model would be resolved against a scope shape + // that is not the one at runtime. Refusing the lowering keeps the + // walker's answer, as it does for `eval` above. + #[cfg(not(feature = "no_custom_syntax"))] + Expr::Custom(..) => { + self.residual_expr(expr); + self.defeated = true; + } + + // Listed rather than matched with `_`, for the reason + // [`Lowering::statement`] gives: a wildcard is what let `eval` + // become a fragment that answered differently from the walker. + // + // These are fragments because none of them can change the shape of + // the scope the slot model resolved its indices against. The + // guarded arms above fall through to here when their guard fails — + // a pool-defeating constant, a literal too long for its operand, a + // call rhai resolves syntactically. + Expr::Coalesce(..) + | Expr::MethodCall(..) + | Expr::Property(..) + | Expr::ThisPtr(..) + | Expr::DynamicConstant(..) + | Expr::FnCall(..) + | Expr::Array(..) + | Expr::Map(..) => self.residual_expr(expr), } } diff --git a/src/grain/format/mod.rs b/src/grain/format/mod.rs index 8572c1107..f8ba70b4c 100644 --- a/src/grain/format/mod.rs +++ b/src/grain/format/mod.rs @@ -85,6 +85,12 @@ mod tail_tag { /// Constant-pool tags, over the subset of `Dynamic` that means the same thing /// in another process. Append only. +/// +/// The whole table is defined on every build even where a restriction feature +/// means nothing can produce a given tag — `no_float` cannot write a `FLOAT`, +/// `no_index` an `ARRAY`. The numbering is the wire format, so it must not +/// shift with the features of whoever compiled the writer. +#[allow(dead_code)] mod constant { pub const UNIT: u8 = 0x00; pub const FALSE: u8 = 0x01; diff --git a/src/grain/mod.rs b/src/grain/mod.rs index baa496076..04c6c864f 100644 --- a/src/grain/mod.rs +++ b/src/grain/mod.rs @@ -42,7 +42,7 @@ //! // The `Scope` is the caller's locals: a script declares are left in it, //! // exactly as `Engine::eval_with_scope` would. //! let mut scope = Scope::new(); -//! let value = Vm::new(&engine).run(&program, &mut scope)?; +//! let value = Vm::new(&engine).eval_with_scope(&mut scope, &program)?; //! //! assert_eq!(value.as_int().unwrap(), 45); //! # Ok::<_, Box>(()) @@ -71,7 +71,7 @@ //! //! // On the device, with no parser and no `AST` in sight. //! let loaded = Program::read(&bytes).expect("written by this build"); -//! let value = Vm::new(&engine).run(&loaded, &mut Scope::new())?; +//! let value = Vm::new(&engine).eval(&loaded)?; //! //! assert_eq!(value.as_int().unwrap(), 42); //! # Ok::<_, Box>(()) diff --git a/src/grain/program.rs b/src/grain/program.rs index 3ffe07542..aed1c749f 100644 --- a/src/grain/program.rs +++ b/src/grain/program.rs @@ -15,7 +15,7 @@ pub(crate) type SharedModule = Shared; /// A program a native function can be handed a way back into. /// -/// [`Vm::run_with_callbacks`](crate::grain::Vm::run_with_callbacks) registers one +/// [`Vm::eval_with_callbacks`](crate::grain::Vm::eval_with_callbacks) registers one /// wrapper per compiled function, and rhai requires a registered function to be /// `'static` — so the program cannot still be borrowing an artifact, and the /// wrappers have to share ownership of it rather than borrow it. @@ -176,6 +176,7 @@ fn unsupported_kind(node: &ASTNode) -> Option<&'static str> { }, ASTNode::Expr(expr) => match expr { Expr::InterpolatedString(..) => "string interpolation", + #[cfg(not(feature = "no_custom_syntax"))] Expr::Custom(..) => "custom syntax", Expr::Map(..) => "a non-constant map literal", _ => return None, @@ -286,7 +287,7 @@ impl<'a> Program<'a> { /// Give up the artifact and share the program, so a native can be handed a /// way back into it. /// - /// What [`Vm::run_with_callbacks`](crate::grain::Vm::run_with_callbacks) takes. + /// What [`Vm::eval_with_callbacks`](crate::grain::Vm::eval_with_callbacks) takes. /// Worth the copy only when [`makes_fn_pointers`](Self::makes_fn_pointers) /// says a pointer can escape. #[must_use] diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 5d0d788d9..472997e02 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -8,9 +8,13 @@ use crate::func::{get_builtin_binary_op_fn, get_builtin_op_assignment_fn}; use crate::packages::string_basic::print_with_func; use crate::types::dynamic::DynamicWriteLock; use crate::types::fn_ptr::FnPtrType; +// `Variant` is only re-exported from the crate root under `internals`, so it +// comes from where it is defined. +use crate::ast::Expr; +use crate::types::dynamic::Variant; use crate::{ - eval::Caches, eval::GlobalRuntimeState, Dynamic, Engine, EvalAltResult, EvalContext, - Expression, Scope, + eval::Caches, eval::GlobalRuntimeState, CallFnOptions, Dynamic, Engine, EvalAltResult, + EvalContext, Scope, }; use crate::{ Array, FnPtr, ImmutableString, Map, NativeCallContext, Position, ThinVec, FUNC_TO_STRING, INT, @@ -401,12 +405,136 @@ impl<'e> Vm<'e> { result } - /// Run a program's main chunk against `scope`, yielding its value. - pub fn run(&mut self, program: &Program, scope: &mut Scope) -> VmResult { + /// Call one compiled function by name, instead of running the whole + /// program. + /// + /// Mirrors [`Engine::call_fn`](crate::Engine::call_fn), including that the + /// program's body runs first — a function usually needs what the top level + /// declared. [`call_fn_with_options`](Self::call_fn_with_options) turns + /// that off. + /// + /// # Errors + /// + /// `ErrorFunctionNotFound` if no compiled function has that name and + /// arity, `ErrorMismatchOutputType` if the result is not a `T`, and + /// whatever the function itself raises. + pub fn call_fn( + &mut self, + scope: &mut Scope, + program: &Program, + name: impl AsRef, + args: impl crate::FuncArgs, + ) -> Result> { + self.call_fn_with_options(CallFnOptions::new(), scope, program, name, args) + } + + /// The same, with rhai's [`CallFnOptions`](crate::CallFnOptions). + /// + /// Three of the five options mean something here: + /// + /// * `eval_ast` runs the program's main chunk before the call, so what the + /// top level declares is in scope for it. On by default, as in rhai. + /// * `rewind_scope` truncates the scope back afterwards. On by default. + /// * `tag` sets the evaluation's custom state. + /// + /// `this_ptr` is accepted and ignored: a compiled function never reads + /// `this`, because the compiler refuses to lower a body that does + /// (`compile/mod.rs`), so a bound `this` could not be observed. A body + /// that uses `this` is still an AST in the program's library and reached + /// through [`Engine::call_fn`](crate::Engine::call_fn) instead. + /// `in_all_namespaces` is likewise ignored — this looks only in the + /// program's own compiled functions. + /// + /// # Errors + /// + /// As [`call_fn`](Self::call_fn). + pub fn call_fn_with_options( + &mut self, + options: CallFnOptions, + scope: &mut Scope, + program: &Program, + name: impl AsRef, + args: impl crate::FuncArgs, + ) -> Result> { + let name = name.as_ref(); + + let mut arg_values = Vec::new(); + args.parse(&mut arg_values); + + let orig_scope_len = scope.len(); + if let Some(tag) = options.tag { + self.global.tag = tag; + } + + if options.eval_ast { + // Run for the scope it leaves behind; the body's own value is not + // what the caller asked for. + let _ = self.eval_with_scope(scope, program)?; + } + + let result = self.call_function(program, name, arg_values, 0, Position::NONE); + + if options.rewind_scope { + scope.rewind(orig_scope_len); + } + + result?.try_cast_result().map_err(|value| { + Box::new(EvalAltResult::ErrorMismatchOutputType( + self.engine + .map_type_name(core::any::type_name::()) + .into(), + self.engine.map_type_name(value.type_name()).into(), + Position::NONE, + )) + }) + } + + /// Evaluate a program's main chunk against `scope`, yielding its value. + /// + /// The scope is the caller's, as it is for + /// [`Engine::eval_ast_with_scope`](crate::Engine::eval_ast_with_scope): + /// what the program declares at the top level is left in it, and what the + /// caller put there beforehand is visible to the program. + /// + /// # Errors + /// + /// Whatever the program raises, and `ErrorRuntime` for a malformed one. + pub fn eval_with_scope(&mut self, scope: &mut Scope, program: &Program) -> VmResult { self.run_with(program, scope, None) } - /// Run a program that hands function pointers to native functions. + /// The same against a scope of its own, for a program that needs none. + /// + /// # Errors + /// + /// As [`eval_with_scope`](Self::eval_with_scope). + pub fn eval(&mut self, program: &Program) -> VmResult { + self.eval_with_scope(&mut Scope::new(), program) + } + + /// Evaluate a program against `scope` for its effects, discarding its value. + /// + /// # Errors + /// + /// As [`eval_with_scope`](Self::eval_with_scope). + pub fn run_with_scope( + &mut self, + scope: &mut Scope, + program: &Program, + ) -> Result<(), Box> { + self.eval_with_scope(scope, program).map(|_| ()) + } + + /// The same against a scope of its own. + /// + /// # Errors + /// + /// As [`eval_with_scope`](Self::eval_with_scope). + pub fn run(&mut self, program: &Program) -> Result<(), Box> { + self.run_with_scope(&mut Scope::new(), program) + } + + /// Evaluate a program that hands function pointers to native functions. /// /// The same run, plus one native wrapper per compiled function registered /// for its duration, so a pointer this program creates resolves when rhai @@ -415,15 +543,23 @@ impl<'e> Vm<'e> { /// module. /// /// Only worth the owned program when [`Program::makes_fn_pointers`] says a - /// pointer can escape; [`run`](Self::run) is otherwise identical and copies - /// nothing. A program that needs this and does not get it still runs — the - /// pointer simply fails to resolve, as `ErrorFunctionNotFound`, at the - /// point the native tries to call it. + /// pointer can escape; [`eval_with_scope`](Self::eval_with_scope) is + /// otherwise identical and copies nothing. A program that needs this and + /// does not get it still runs — the pointer simply fails to resolve, as + /// `ErrorFunctionNotFound`, at the point the native tries to call it. /// /// Read the `callback` module before relying on it: a crossing is slower than the /// walker, and a *capturing* closure handed to a native that binds `this` /// arrives with its arguments rotated. - pub fn run_with_callbacks(&mut self, program: &SharedProgram, scope: &mut Scope) -> VmResult { + /// + /// Named `eval_` rather than `run_` because it yields the program's value; + /// rhai has no `Engine` method to mirror here, so the crate's own rule is + /// the one that applies. + /// + /// # Errors + /// + /// As [`eval_with_scope`](Self::eval_with_scope). + pub fn eval_with_callbacks(&mut self, scope: &mut Scope, program: &SharedProgram) -> VmResult { let wrappers = (!program.functions().is_empty()).then(|| callback::wrappers(program).into()); self.run_with(program, scope, wrappers) @@ -2348,21 +2484,28 @@ impl<'e> Vm<'e> { .ok_or_else(|| malformed(format!("no residual {index}")))?; let rewind_scope = tag == code::tag::EVAL_AST; - let mut context = EvalContext::new( - self.engine, - &mut self.global, - &mut self.caches, - scope, - None, - ); - - // The deprecation marker on this method means "volatile, - // may change", not "going away" — it is the only public - // route from outside the crate to rhai's own walker, and - // total language coverage rests on it. - #[allow(deprecated)] - let value = - context.eval_expression_tree_raw(&Expression::from(expr), rewind_scope)?; + // Straight to the walker's own entry points rather than + // through `EvalContext::eval_expression_tree_raw`, which + // is the same two calls behind a shim that only exists + // under `custom_syntax`. Total language coverage rests on + // this, so it must not depend on a feature. + let value = match expr { + Expr::Stmt(block) => self.engine.eval_stmt_block( + &mut self.global, + &mut self.caches, + scope, + None, + block.statements(), + rewind_scope, + ), + expr => self.engine.eval_expr( + &mut self.global, + &mut self.caches, + scope, + None, + expr, + ), + }?; self.stack.push(value); } diff --git a/tests/grain/allocation.rs b/tests/grain/allocation.rs index 304fe722c..22c8d6f4e 100644 --- a/tests/grain/allocation.rs +++ b/tests/grain/allocation.rs @@ -183,7 +183,12 @@ while i < 64 { total; "#; +// `SCRIPT` and `follow.rhai` both use floats, and the second is checked in +// byte-identical to the script the 24-bytes-per-source-byte figure came from — +// rewriting it to suit a build would make the number mean something else. So +// the measurement is a default-build one, and says so. #[test] +#[cfg(not(feature = "no_float"))] fn allocation_footprint() { let engine = Engine::new(); diff --git a/tests/grain/callback.rs b/tests/grain/callback.rs index ee5f61dbe..c53a2e689 100644 --- a/tests/grain/callback.rs +++ b/tests/grain/callback.rs @@ -18,7 +18,7 @@ fn run(engine: &Engine, source: &str) -> Result { let ast = engine.compile(source).map_err(|err| format!("{err:?}"))?; let program = Compiler::new().compile(&ast).into_shared(); Vm::new(engine) - .run_with_callbacks(&program, &mut Scope::new()) + .eval_with_callbacks(&mut Scope::new(), &program) .map(|value| format!("{value:?}")) .map_err(|err| format!("{err:?}")) } @@ -183,7 +183,7 @@ fn a_callback_costs_more_call_levels() { #[test] fn without_the_wrappers_the_pointer_does_not_resolve() { - // The whole reason `run_with_callbacks` exists. A plain `run` leaves rhai + // The whole reason `eval_with_callbacks` exists. A plain eval leaves rhai // nowhere to look, and the failure is a lookup failure rather than // anything worse. let engine = corpus::engine(); @@ -191,6 +191,6 @@ fn without_the_wrappers_the_pointer_does_not_resolve() { let ast = engine.compile(source).unwrap(); let program = Compiler::new().compile(&ast); - let err = Vm::new(&engine).run(&program, &mut Scope::new()).unwrap_err(); + let err = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).unwrap_err(); assert!(format!("{err:?}").contains("ErrorFunctionNotFound"), "{err}"); } diff --git a/tests/grain/corpus/generate.rs b/tests/grain/corpus/generate.rs index 6f5d5748b..02d84b307 100644 --- a/tests/grain/corpus/generate.rs +++ b/tests/grain/corpus/generate.rs @@ -410,6 +410,10 @@ impl Generator { fn atom(&mut self) -> String { match self.rng.below(8) { 0..=2 => format!("{}", self.rng.below(64)), + // A float literal is not syntax under `no_float` — rhai reads the + // `.` as a property access — so the whole script would fail to + // parse and test nothing. + #[cfg(not(feature = "no_float"))] 3 => format!("{}.{}", self.rng.below(8), self.rng.below(8)), 4 => format!("\"s{}\"", self.rng.below(8)), 5 => self.rng.one_of(&["true", "false"]).to_string(), diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index 66fef3802..958d9b483 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -86,6 +86,38 @@ const fn case(name: &'static str, source: &'static str) -> Case { Case { name, source } } +/// Whether a corpus case exercises anything on this build. +/// +/// A restriction feature removes the syntax outright — rhai will not parse a +/// capturing closure under `no_closure`, or a float literal under `no_float` — +/// so the case tests nothing here, and both sides agreeing on the parse failure +/// would be an empty agreement rather than a passing one. +/// +/// Lives here rather than in one harness because every harness that walks +/// [`CASES`] needs the same answer. +#[must_use] +pub fn applies_to_this_build(name: &str) -> bool { + #[cfg(feature = "no_closure")] + if name.starts_with("closure_") || name.starts_with("is_shared") { + return false; + } + #[cfg(feature = "no_module")] + if name.starts_with("import_") || name.starts_with("export_") { + return false; + } + // No shared prefix to key on: a float literal is incidental to most of + // these, which are about interpolation, ranges and operator errors. + #[cfg(feature = "no_float")] + if matches!( + name, + "float_arithmetic" | "mixed_numeric" | "interpolation_of_every_type" | "switch_float_in_range" | "error_operator_undefined_for_types" | "error_op_assign_undefined_for_types" + ) { + return false; + } + let _ = name; + true +} + pub const CASES: &[Case] = &[ // --- values and operators ------------------------------------------- case("int_arithmetic", "let a = 7; let b = 3; a * b - a / b + a % b"), diff --git a/tests/grain/differential.rs b/tests/grain/differential.rs index 51ef22aaa..b203952e5 100644 --- a/tests/grain/differential.rs +++ b/tests/grain/differential.rs @@ -51,9 +51,9 @@ fn run_vm(engine: &Engine, source: &str) -> Outcome { // against a configuration nobody would ship. if program.makes_fn_pointers() { let program = program.into_shared(); - Vm::new(engine).run_with_callbacks(&program, &mut scope) + Vm::new(engine).eval_with_callbacks(&mut scope, &program) } else { - Vm::new(engine).run(&program, &mut scope) + Vm::new(engine).eval_with_scope(&mut scope, &program) } .map(|value| format!("{value:?}")) .map_err(|err| format!("{err:?}")) @@ -68,7 +68,7 @@ fn vm_agrees_with_rhai() { let mut failures = Vec::new(); - for case in corpus::CASES { + for case in corpus::CASES.iter().filter(|c| applies_to_this_build(c.name)) { let stock = run_stock(&engine, case.source); let vm = run_vm(&engine, case.source); @@ -81,7 +81,8 @@ fn vm_agrees_with_rhai() { } } - assert!(failures.is_empty(), "{} of {} corpus scripts diverged:{}", failures.len(), corpus::CASES.len(), failures.join(""),); + let applicable = corpus::CASES.iter().filter(|c| applies_to_this_build(c.name)).count(); + assert!(failures.is_empty(), "{} of {applicable} corpus scripts diverged:{}", failures.len(), failures.join(""),); } /// The corpus is only worth anything if the comparison can actually fail. @@ -107,6 +108,7 @@ fn every_corpus_script_parses() { let broken: Vec<_> = corpus::CASES .iter() + .filter(|case| applies_to_this_build(case.name)) .filter_map(|case| engine.compile(case.source).err().map(|err| format!("\n {}: {err}", case.name))) .collect(); @@ -115,22 +117,10 @@ fn every_corpus_script_parses() { /// Whether a corpus case exercises anything on this build. /// -/// Distinct from [`MAY_FRAGMENT`], which is a tolerance. A restriction feature -/// removes the syntax outright — rhai will not parse a capturing closure under -/// `no_closure` — so the case tests nothing here, and both sides agreeing on -/// the parse failure would be an empty agreement rather than a passing one. -fn applies_to_this_build(name: &str) -> bool { - #[cfg(feature = "no_closure")] - if name.starts_with("closure_") || name.starts_with("is_shared") { - return false; - } - #[cfg(feature = "no_module")] - if name.starts_with("import_") || name.starts_with("export_") { - return false; - } - let _ = name; - true -} +/// Distinct from [`MAY_FRAGMENT`], which is a tolerance: this says the syntax +/// is not in the language on this build at all. Defined beside the cases, in +/// `corpus`, because every harness that walks them needs the same answer. +use corpus::applies_to_this_build; /// A case that errors unintentionally is nearly as weak as one that does not /// parse: both sides agree on the failure, and the machinery the case was @@ -258,7 +248,7 @@ fn residual_census() { } } - println!("\n{} of {} scripts fully lowered, {total_nodes} AST nodes still in fragments", at_zero.len(), corpus::CASES.len(),); + println!("\n{} of {applicable} scripts fully lowered, {total_nodes} AST nodes still in fragments", at_zero.len(),); println!("\nfully lowered: {}", at_zero.join(", ")); println!("\nremaining:"); remaining.sort_by_key(|(_, count)| std::cmp::Reverse(*count)); diff --git a/tests/grain/format.rs b/tests/grain/format.rs index 31c3770a1..63a82367a 100644 --- a/tests/grain/format.rs +++ b/tests/grain/format.rs @@ -36,9 +36,9 @@ fn run(engine: &Engine, program: Program) -> Outcome { let mut scope = Scope::new(); let result = if program.makes_fn_pointers() { let program = program.into_shared(); - Vm::new(engine).run_with_callbacks(&program, &mut scope) + Vm::new(engine).eval_with_callbacks(&mut scope, &program) } else { - Vm::new(engine).run(&program, &mut scope) + Vm::new(engine).eval_with_scope(&mut scope, &program) }; snapshot(&scope, result) @@ -104,8 +104,11 @@ fn the_round_trip_covers_something_worth_covering() { // One per construct the encoder has a branch for, so a branch that stops // working names itself. for required in [ - "int_arithmetic", // Call with an operator token - "float_arithmetic", // a float constant, whose width the ABI pins + "int_arithmetic", // Call with an operator token + // A float constant, whose width the ABI pins — and which `no_float` + // removes from the language, so there is no branch left to cover. + #[cfg(not(feature = "no_float"))] + "float_arithmetic", "shadowing_nested", // DeclareLocal and UnwindTo "while_loop", // jumps, Tick, AssignLocal with an op "loop_break_value", // backpatched jumps @@ -150,107 +153,119 @@ fn golden_scope() -> Scope<'static> { /// the question it asks is whether that was deliberate. If it was, regenerate: /// /// ```text -/// REGENERATE_GOLDEN=1 cargo test --features grain --test mod golden +/// REGENERATE_GOLDEN=1 cargo test --features grain --test grain golden /// ``` /// /// and bump `VERSION` if an older reader would *misread* the new bytes rather /// than reject them — the rule is at `src/format/mod.rs:56`. #[test] fn a_golden_artifact_written_by_an_older_build_still_runs() { - let engine = corpus::engine(); - let source = std::fs::read_to_string(GOLDEN_SOURCE).expect("the golden source is checked in"); - let ast = engine.compile(&source).expect("the golden source must parse"); - let program = Compiler::new().compile(&ast); - assert_eq!(program.residual_count(), 0, "the golden source must lower whole, or the artifact covers less than it claims: {:?}", program.first_unsupported(),); - - if std::env::var_os("REGENERATE_GOLDEN").is_some() { - let bytes = program.write().expect("the golden source must be writable"); - std::fs::write(GOLDEN_ARTIFACT, &bytes).expect("must write the artifact"); - println!("wrote {} bytes to {GOLDEN_ARTIFACT}", bytes.len()); + // The fixture is one build's bytes and its source is that build's source, + // which uses floats. `no_float` cannot parse it, so there is nothing here + // to check — the same reason the ABI guard skips below, reached earlier. + #[cfg(feature = "no_float")] + { + println!("skipped: the golden source uses floats, which this build has no syntax for"); return; } - let bytes = std::fs::read(GOLDEN_ARTIFACT).expect("the golden artifact is checked in"); - let loaded = match Program::read(&bytes) { - Ok(loaded) => loaded, - // The header records the ABI the fixture was written under, and a build - // with different numeric widths or restriction flags refuses it *by - // design* — that refusal is what `abi.rs` is for. The fixture is one - // build's bytes, so it can only be checked on that build; anywhere else - // this would be testing the ABI guard rather than the encoding. - Err(err) if format!("{err}").contains("written with") => { - println!("skipped: the golden fixture is a default-build artifact ({err})"); + #[cfg(not(feature = "no_float"))] + { + let engine = corpus::engine(); + let source = std::fs::read_to_string(GOLDEN_SOURCE).expect("the golden source is checked in"); + let ast = engine.compile(&source).expect("the golden source must parse"); + let program = Compiler::new().compile(&ast); + assert_eq!(program.residual_count(), 0, "the golden source must lower whole, or the artifact covers less than it claims: {:?}", program.first_unsupported(),); + + if std::env::var_os("REGENERATE_GOLDEN").is_some() { + let bytes = program.write().expect("the golden source must be writable"); + std::fs::write(GOLDEN_ARTIFACT, &bytes).expect("must write the artifact"); + println!("wrote {} bytes to {GOLDEN_ARTIFACT}", bytes.len()); return; } - Err(err) => panic!( - "the golden artifact no longer loads: {err}\n\ + + let bytes = std::fs::read(GOLDEN_ARTIFACT).expect("the golden artifact is checked in"); + let loaded = match Program::read(&bytes) { + Ok(loaded) => loaded, + // The header records the ABI the fixture was written under, and a build + // with different numeric widths or restriction flags refuses it *by + // design* — that refusal is what `abi.rs` is for. The fixture is one + // build's bytes, so it can only be checked on that build; anywhere else + // this would be testing the ABI guard rather than the encoding. + Err(err) if format!("{err}").contains("written with") => { + println!("skipped: the golden fixture is a default-build artifact ({err})"); + return; + } + Err(err) => panic!( + "the golden artifact no longer loads: {err}\n\ The format moved. If that was deliberate, regenerate the fixture with \ - `REGENERATE_GOLDEN=1 cargo test --features grain --test mod golden`.", - ), - }; + `REGENERATE_GOLDEN=1 cargo test --features grain --test grain golden`.", + ), + }; - // A fixture only pins what it contains, and narrowing one while editing the - // source is easy and silent. These are read off the *artifact*, so they say - // what the encoder branch coverage actually is rather than what the source - // looks like it should give. - let kinds: std::collections::BTreeSet = rhai::grain::bytecode::disassemble(loaded.code()) - .map(|(_, op)| format!("{op:?}").split(['(', ' ', '{']).next().unwrap_or_default().to_string()) - .collect(); - for required in [ - "Chain", // a chain record, with all three of its roots - "CallRef", // and both by-reference call forms - "Rotate", // which only a named receiver needs - "LoadNamed", // the caller's variable, flat - "LoadSharedNamed", // and as the cell a capture binds - "MakeClosure", // a function pointer to a compiled chunk - "Curry", // with what it captured bound onto it - "MakeArray", // a literal the optimizer could not fold - "MakeMap", // and its template-plus-pairs cousin - "CheckSize", // the per-element size check beside it - "PushHandler", // a handler region, whose catch variable is pooled - "Throw", // - "IterNext", // an iterator, and the two-edged instruction - "InterpolateAppend", // a string built a segment at a time - ] { - assert!( - kinds.contains(required), - "the golden no longer contains `{required}`, so its encoder branch \ + // A fixture only pins what it contains, and narrowing one while editing the + // source is easy and silent. These are read off the *artifact*, so they say + // what the encoder branch coverage actually is rather than what the source + // looks like it should give. + let kinds: std::collections::BTreeSet = rhai::grain::bytecode::disassemble(loaded.code()) + .map(|(_, op)| format!("{op:?}").split(['(', ' ', '{']).next().unwrap_or_default().to_string()) + .collect(); + for required in [ + "Chain", // a chain record, with all three of its roots + "CallRef", // and both by-reference call forms + "Rotate", // which only a named receiver needs + "LoadNamed", // the caller's variable, flat + "LoadSharedNamed", // and as the cell a capture binds + "MakeClosure", // a function pointer to a compiled chunk + "Curry", // with what it captured bound onto it + "MakeArray", // a literal the optimizer could not fold + "MakeMap", // and its template-plus-pairs cousin + "CheckSize", // the per-element size check beside it + "PushHandler", // a handler region, whose catch variable is pooled + "Throw", // + "IterNext", // an iterator, and the two-edged instruction + "InterpolateAppend", // a string built a segment at a time + ] { + assert!( + kinds.contains(required), + "the golden no longer contains `{required}`, so its encoder branch \ is unpinned again — put it back or say why it went", - ); - } - assert!( - kinds.len() >= 35, - "the golden covers only {} instruction kinds, which is narrower than it \ + ); + } + assert!( + kinds.len() >= 35, + "the golden covers only {} instruction kinds, which is narrower than it \ was written to be: {kinds:?}", - kinds.len(), - ); + kinds.len(), + ); - let walked = { - let mut scope = golden_scope(); - let result = engine.eval_ast_with_scope::(&mut scope, &ast); - snapshot(&scope, result) - }; - let ran = { - let mut scope = golden_scope(); - // Whether the program can hand a pointer to a native is read back off - // the bytes, so how it must be run is part of what is being checked. - let result = if loaded.makes_fn_pointers() { - let loaded = loaded.into_shared(); - Vm::new(&engine).run_with_callbacks(&loaded, &mut scope) - } else { - Vm::new(&engine).run(&loaded, &mut scope) + let walked = { + let mut scope = golden_scope(); + let result = engine.eval_ast_with_scope::(&mut scope, &ast); + snapshot(&scope, result) + }; + let ran = { + let mut scope = golden_scope(); + // Whether the program can hand a pointer to a native is read back off + // the bytes, so how it must be run is part of what is being checked. + let result = if loaded.makes_fn_pointers() { + let loaded = loaded.into_shared(); + Vm::new(&engine).eval_with_callbacks(&mut scope, &loaded) + } else { + Vm::new(&engine).eval_with_scope(&mut scope, &loaded) + }; + snapshot(&scope, result) }; - snapshot(&scope, result) - }; - assert_eq!( - ran, walked, - "the golden artifact no longer means what its source means.\n\ + assert_eq!( + ran, walked, + "the golden artifact no longer means what its source means.\n\ The format moved without the reader noticing, which is the failure this \ fixture exists to catch. If the change was deliberate, regenerate with \ - `REGENERATE_GOLDEN=1 cargo test --features grain --test mod golden`.", - ); - assert!(ran.result.is_ok(), "the golden must produce a value, not an error: {ran:?}",); + `REGENERATE_GOLDEN=1 cargo test --features grain --test grain golden`.", + ); + assert!(ran.result.is_ok(), "the golden must produce a value, not an error: {ran:?}",); + } } /// A chain rooted at a caller's variable, which the corpus cannot cover. @@ -283,7 +298,7 @@ fn a_chain_rooted_at_a_name_survives_the_round_trip() { let mut loaded = Scope::new(); seed(&mut loaded); let actual = { - let out = Vm::new(&engine).run(&reloaded, &mut loaded); + let out = Vm::new(&engine).eval_with_scope(&mut loaded, &reloaded); snapshot(&loaded, out) }; @@ -371,14 +386,21 @@ fn a_function_the_compiler_cannot_lower_refuses_to_write() { /// receiver that is not on the operand stack, so a corrupted one is read /// against a different depth than any other call's. fn sample(engine: &Engine) -> Vec { - let ast = engine - .compile( - "let a = 1; let b = 2.5; while a < 10 { a += 1 } \ - let c = [a]; push(c, b); push(caller_supplied, a); \ - caller_supplied[0] = a; \ - switch a { 1 => \"one\", 0..=20 => \"some\", _ => \"many\" }", - ) - .expect("must compile"); + // A float in the constant pool is part of what this covers, and `no_float` + // has no float to put there. The rest of the shape — a loop, an array, a + // caller variable, an indexed write, a switch — is the same either way. + #[cfg(not(feature = "no_float"))] + const SECOND: &str = "2.5"; + #[cfg(feature = "no_float")] + const SECOND: &str = "2"; + + let source = format!( + "let a = 1; let b = {SECOND}; while a < 10 {{ a += 1 }} \ + let c = [a]; push(c, b); push(caller_supplied, a); \ + caller_supplied[0] = a; \ + switch a {{ 1 => \"one\", 0..=20 => \"some\", _ => \"many\" }}" + ); + let ast = engine.compile(&source).expect("must compile"); Compiler::new().compile(&ast).write().expect("the sample must be writable") } @@ -505,7 +527,7 @@ fn no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk() { loaded += 1; program.verify().expect("read must not return a chunk that fails verification"); // The result is free to be anything; not crashing is the claim. - let _ = Vm::new(&engine).run(&program, &mut Scope::new()); + let _ = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program); } } } @@ -537,7 +559,7 @@ fn a_stripped_program_reports_an_address_the_host_can_resolve() { assert!(device.positions().is_stripped(), "a stripped artifact must not carry positions",); let mut vm = Vm::new(&engine); - let error = vm.run(&device, &mut Scope::new()).expect_err("dividing by zero must fail"); + let error = vm.eval_with_scope(&mut Scope::new(), &device).expect_err("dividing by zero must fail"); let address = vm.fault_pc().expect("a failed run must name an instruction"); // Host: resolve what came back. diff --git a/tests/grain/fuzz.rs b/tests/grain/fuzz.rs index bcc59a684..0a7210c2e 100644 --- a/tests/grain/fuzz.rs +++ b/tests/grain/fuzz.rs @@ -128,7 +128,7 @@ fn mutated_artifacts_load_or_fail_but_never_misbehave() { // And must then run without taking the process down. The result is // free to be anything at all. - let _ = Vm::new(&engine).run(&program, &mut Scope::new()); + let _ = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program); } } @@ -156,7 +156,7 @@ fn arbitrary_bytes_never_load_into_something_that_misbehaves() { if let Ok(program) = Program::read(&bytes) { assert!(program.verify().is_ok(), "unverified: {}", hex(&bytes)); - let _ = Vm::new(&engine).run(&program, &mut Scope::new()); + let _ = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program); } } } @@ -294,9 +294,9 @@ fn agree_unoptimised(source: &str) -> bool { let mut vm_scope = Scope::new(); let ours = if program.makes_fn_pointers() { let program = program.into_shared(); - Vm::new(&plain).run_with_callbacks(&program, &mut vm_scope) + Vm::new(&plain).eval_with_callbacks(&mut vm_scope, &program) } else { - Vm::new(&plain).run(&program, &mut vm_scope) + Vm::new(&plain).eval_with_scope(&mut vm_scope, &program) }; snapshot(&walker_scope, walked) == snapshot(&vm_scope, ours) @@ -382,9 +382,9 @@ fn generated_scripts_agree_with_the_walker() { let mut vm_scope = Scope::new(); let ours = if program.makes_fn_pointers() { let program = program.into_shared(); - Vm::new(&engine).run_with_callbacks(&program, &mut vm_scope) + Vm::new(&engine).eval_with_callbacks(&mut vm_scope, &program) } else { - Vm::new(&engine).run(&program, &mut vm_scope) + Vm::new(&engine).eval_with_scope(&mut vm_scope, &program) }; let ours = snapshot(&vm_scope, ours); diff --git a/tests/grain/limits.rs b/tests/grain/limits.rs index b54fbe6e5..c29c84911 100644 --- a/tests/grain/limits.rs +++ b/tests/grain/limits.rs @@ -32,7 +32,7 @@ fn run_vm(engine: &Engine, source: &str) -> Result> // `track_operation`, and the tests below would hang rather than fail. assert!(program.main().ops(program.code()).any(|(_, op)| op == Op::Tick), "{source:?} lowered to a loop with no operation tick",); - Vm::new(engine).run(&program, &mut Scope::new()) + Vm::new(engine).eval_with_scope(&mut Scope::new(), &program) } #[test] @@ -100,7 +100,7 @@ fn a_loop_with_its_tick_removed_still_hits_the_limit() { let tickless = Program::read(&bytes).expect("still a valid artifact"); assert!(!tickless.main().ops(tickless.code()).any(|(_, op)| op == Op::Tick), "the tick should be gone, or this tests nothing",); - let err = Vm::new(&engine).run(&tickless, &mut Scope::new()).expect_err("a tickless loop must still be stopped"); + let err = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &tickless).expect_err("a tickless loop must still be stopped"); assert!(matches!(*err, EvalAltResult::ErrorTooManyOperations(..)), "expected ErrorTooManyOperations, got {err:?}",); } @@ -137,7 +137,7 @@ fn interpolation_respects_the_string_limit() { assert_eq!(program.residual_count(), 0, "must be lowered, not walked"); let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect_err("the walker must refuse it"); - let vm = Vm::new(&engine).run(&program, &mut Scope::new()).expect_err("and so must the VM"); + let vm = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect_err("and so must the VM"); assert!(matches!(*vm, EvalAltResult::ErrorDataTooLarge(..)), "got {vm:?}",); assert_eq!(format!("{vm:?}"), format!("{walker:?}"), "including the position of the segment that went over",); @@ -154,7 +154,7 @@ fn ticking_does_not_disturb_a_bounded_loop() { let ast = engine.compile(source).expect("must compile"); let program = Compiler::new().compile(&ast); - let vm = Vm::new(&engine).run(&program, &mut Scope::new()).expect("bounded loop must finish"); + let vm = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect("bounded loop must finish"); let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect("bounded loop must finish under rhai too"); diff --git a/tests/grain/scope.rs b/tests/grain/scope.rs index 36af79e4f..a561f1f54 100644 --- a/tests/grain/scope.rs +++ b/tests/grain/scope.rs @@ -68,7 +68,7 @@ fn agree_with(engine: &Engine, source: &str, build: impl Fn(&mut Scope), writabl let mut run = Scope::new(); build(&mut run); let actual = { - let result = Vm::new(engine).run(&program, &mut run); + let result = Vm::new(engine).eval_with_scope(&mut run, &program); capture(&run, result) }; @@ -398,13 +398,13 @@ fn a_global_module_constant_resolves() { let program = Compiler::new().compile(&ast); assert_eq!(program.residual_count(), 0); - let value = Vm::new(&engine).run(&program, &mut Scope::new()).expect("a module constant must resolve"); + let value = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect("a module constant must resolve"); assert_eq!(value.as_int().unwrap(), 256); // And writing to one is refused, because it is a value and not a place. let ast = engine.compile("CHANNELS = 1").expect("must compile"); let program = Compiler::new().compile(&ast); - let err = Vm::new(&engine).run(&program, &mut Scope::new()).expect_err("a module constant is not assignable"); + let err = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect_err("a module constant is not assignable"); assert!(matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), "got {err:?}",); } @@ -421,6 +421,7 @@ fn a_global_module_constant_resolves() { /// as one block. These cases are here to keep it refusing: an `import` that /// started lowering again would put the divergence straight back. #[test] +#[cfg(not(feature = "no_module"))] fn an_import_keeps_the_walkers_answer() { let mut engine = corpus::engine(); @@ -446,6 +447,50 @@ fn an_import_keeps_the_walkers_answer() { agree_with(&engine, r#"fn via_kit() { import "kit" as k; k::double(3) } via_kit()"#, |_| {}, true); } +/// `eval` costs the body its lowering, and must. +/// +/// It runs against the caller's scope and can declare into it, and the AST +/// does not say so. The slot model resolves indices against the scope shape it +/// can see, so a read after one looks in the wrong place — +/// `eval("let x = 40"); x + 2` found no `x` where the walker found 40. +/// +/// Written as a divergence rather than a missing feature, which is what makes +/// it worth a test: it used to become an ordinary fragment and answer wrongly. +#[test] +fn eval_keeps_the_walkers_answer() { + let engine = corpus::engine(); + + for source in [ + // The case that diverged: what `eval` declares outlives it. + r#"eval("let x = 40"); x + 2"#, + r#"let x = 1; eval("x = 99"); x"#, + r#"eval("1 + 1")"#, + r#"eval("let y = 7; y * 3")"#, + ] { + agree_with(&engine, source, |_| {}, false); + } +} + +/// The same for custom syntax, which reaches the caller's scope through an +/// `EvalContext` and is likewise invisible to the slot model. +#[test] +#[cfg(not(feature = "no_custom_syntax"))] +fn custom_syntax_keeps_the_walkers_answer() { + let mut engine = corpus::engine(); + engine + .register_custom_syntax(["declare", "$ident$", "=", "$int$"], true, |context, inputs| { + let name = inputs[0].get_string_value().unwrap().to_string(); + let value = inputs[1].get_literal_value::().unwrap(); + context.scope_mut().push(name, value); + Ok(Dynamic::UNIT) + }) + .expect("the custom syntax must register"); + + for source in [r#"declare foo = 41; foo + 1"#, r#"declare bar = 5; 10"#] { + agree_with(&engine, source, |_| {}, false); + } +} + /// The first of the three, and the one a VM would most plausibly skip: a /// resolver the host registered through `Engine::on_var` sees the name before /// the scope does. @@ -471,12 +516,12 @@ fn a_variable_resolver_is_consulted_first() { scope.push("ordinary", 5 as INT); let program = compile(&engine, "injected + ordinary"); - let value = Vm::new(&engine).run(&program, &mut scope.clone()).expect("both must resolve"); + let value = Vm::new(&engine).eval_with_scope(&mut scope.clone(), &program).expect("both must resolve"); assert_eq!(value.as_int().unwrap(), 104); // A resolver hands back a value rather than a place, so it is read-only. let program = compile(&engine, "injected = 1"); - let err = Vm::new(&engine).run(&program, &mut scope.clone()).expect_err("a resolved variable is not assignable"); + let err = Vm::new(&engine).eval_with_scope(&mut scope.clone(), &program).expect_err("a resolved variable is not assignable"); assert!(matches!(*err, rhai::EvalAltResult::ErrorAssignmentToConstant(..)), "got {err:?}",); // And the walker agrees about all of it. @@ -504,7 +549,7 @@ fn a_resolver_that_grows_the_scope_forces_a_search() { assert_eq!(program.residual_count(), 0); let mut scope = Scope::new(); - let value = Vm::new(&engine).run(&program, &mut scope).expect("must run"); + let value = Vm::new(&engine).eval_with_scope(&mut scope, &program).expect("must run"); let mut walked = Scope::new(); let expected = engine.eval_ast_with_scope::(&mut walked, &ast).expect("the walker must run it too"); @@ -547,7 +592,7 @@ fn a_resolved_receiver_is_not_the_scope_entry_it_shadows() { let mut run = Scope::new(); start(&mut run); - let ours = Vm::new(&engine).run(&program, &mut run); + let ours = Vm::new(&engine).eval_with_scope(&mut run, &program); // Read-only all the way through, so rhai refuses the call outright rather // than mutating a copy — which is a sharper thing to agree on. @@ -575,7 +620,7 @@ fn a_closure_pointer_is_late_bound() { let program = Compiler::new().compile(&ast); assert_eq!(program.residual_count(), 0, "the closure must lower"); - let ours = Vm::new(&engine).run(&program, &mut Scope::new()).expect("must run"); + let ours = Vm::new(&engine).eval_with_scope(&mut Scope::new(), &program).expect("must run"); let walker = engine.eval_ast_with_scope::(&mut Scope::new(), &ast).expect("must run under rhai too"); let (ours, walker) = (format!("{ours:?}"), format!("{walker:?}")); @@ -614,6 +659,50 @@ fn a_compiled_function_can_be_called_by_name() { assert_eq!(value.as_int().unwrap(), 11); } +/// The same call through the API a host actually reaches for, which mirrors +/// [`Engine::call_fn`]. +/// +/// The differences from `call_function` are the ones rhai's own `call_fn` has: +/// arguments as a tuple rather than a `Vec`, a typed result, and the +/// program's body run first — which is what sets up anything the call needs. +#[test] +fn call_fn_mirrors_the_engines() { + let engine = corpus::engine(); + let ast = engine.compile("fn add(a, b) { a + b } let marker = 10; 0").expect("must compile"); + let program = Compiler::new().compile(&ast); + + let mut vm = Vm::new(&engine); + let mut scope = Scope::new(); + + let sum: INT = vm.call_fn(&mut scope, &program, "add", (2 as INT, 3 as INT)).expect("must call"); + assert_eq!(sum, 5); + // `eval_ast` ran the body, and `rewind_scope` took back what it declared. + // Both default to on, as they do in rhai. + assert_eq!(scope.len(), 0, "the default rewinds what the body declared"); + + let keep = rhai::CallFnOptions::new().rewind_scope(false); + let sum: INT = vm.call_fn_with_options(keep, &mut scope, &program, "add", (4 as INT, 5 as INT)).expect("must call"); + assert_eq!(sum, 9); + assert_eq!(scope.len(), 1, "`marker` outlives the call now"); + assert_eq!(scope.get_value::("marker").unwrap(), 10); + + // Skipping the body is what a caller does when the function needs nothing + // from it — the common case, and the one that costs nothing. + let mut fresh = Scope::new(); + let skip = rhai::CallFnOptions::new().eval_ast(false); + let sum: INT = vm.call_fn_with_options(skip, &mut fresh, &program, "add", (1 as INT, 1 as INT)).expect("must call"); + assert_eq!(sum, 2); + assert_eq!(fresh.len(), 0, "the body never ran, so it declared nothing"); + + // A result that is not the type asked for is an error, not a panic. + let err = vm.call_fn::(&mut scope, &program, "add", (1 as INT, 2 as INT)).expect_err("an INT is not a String"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorMismatchOutputType(..)), "got {err:?}",); + + // And a missing name still misses. + let err = vm.call_fn::(&mut scope, &program, "nope", (1 as INT,)).expect_err("no such function"); + assert!(matches!(*err, rhai::EvalAltResult::ErrorFunctionNotFound(..)), "got {err:?}",); +} + /// The flag a host uses to decide whether a program has to be owned. /// /// Registering compiled functions so a native can call one back requires a @@ -663,7 +752,7 @@ fn a_program_reading_caller_state_can_be_written() { let mut scope = Scope::new(); scope.push("brightness", 5 as INT); scope.push("mode", "chase".to_string()); - let value = Vm::new(&engine).run(&reloaded, &mut scope).expect("must run"); + let value = Vm::new(&engine).eval_with_scope(&mut scope, &reloaded).expect("must run"); assert_eq!(value.as_int().unwrap(), 15); } From b8cfd74e852c28bf6e57e04860a597840cbdf6fb Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 12:31:58 -0700 Subject: [PATCH 15/33] install a program's environment for the whole of call_fn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A program's library, source and module resolver were pushed onto `global` around the main chunk only, so a function reached through `call_fn` ran without them. Anything the compiler could not lower stays an AST in the library, and rhai finds it only in `global.lib` — so a lowered function calling such a helper reported `ErrorFunctionNotFound`, or, when the name was also a rhai built-in, silently answered the built-in instead. Hoist the push and restore into `with_environment` and wrap both halves of `call_fn_with_options` in it. `run_with` splits into that plus `run_main`, which is the same body it always had. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/vm/mod.rs | 82 +++++++++++++++++++------- tests/grain/call_fn.rs | 127 +++++++++++++++++++++++++++++++++++++++++ tests/mod.rs | 1 + 3 files changed, 188 insertions(+), 22 deletions(-) create mode 100644 tests/grain/call_fn.rs diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 472997e02..2f7dc07f7 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -220,6 +220,18 @@ fn place<'a>( .ok_or_else(|| Box::new(EvalAltResult::ErrorDataRace(name.to_string(), pos))) } +/// Turn the two control-flow errors back into the value they carry. +/// +/// Rhai unwinds `return` and `exit` as errors rather than returning them; +/// `eval_global_statements` is where they turn back into values, and anything +/// entering a program from outside has to do the same. +fn unwind_exit(result: VmResult) -> VmResult { + result.or_else(|err| match *err { + EvalAltResult::Return(out, ..) | EvalAltResult::Exit(out, ..) => Ok(out), + _ => Err(err), + }) +} + fn missing(name: &str, pos: Position) -> Box { Box::new(EvalAltResult::ErrorVariableNotFound(name.to_string(), pos)) } @@ -466,13 +478,23 @@ impl<'e> Vm<'e> { self.global.tag = tag; } - if options.eval_ast { - // Run for the scope it leaves behind; the body's own value is not - // what the caller asked for. - let _ = self.eval_with_scope(scope, program)?; - } - - let result = self.call_function(program, name, arg_values, 0, Position::NONE); + // The program's environment stays installed for the *call*, not only for + // the main chunk that may precede it: the function being called can + // reach whatever the compiler left rhai to interpret, and rhai looks for + // it in `global.lib`. + let mut evaluated = Ok(Dynamic::UNIT); + let result = self.with_environment(program, None, |vm| { + if options.eval_ast { + // Run for the scope it leaves behind; the body's own value is + // not what the caller asked for. + evaluated = unwind_exit(vm.run_main(program, scope)); + if evaluated.is_err() { + return Ok(Dynamic::UNIT); + } + } + vm.call_function(program, name, arg_values, 0, Position::NONE) + }); + evaluated?; if options.rewind_scope { scope.rewind(orig_scope_len); @@ -571,6 +593,22 @@ impl<'e> Vm<'e> { scope: &mut Scope, wrappers: Option, ) -> VmResult { + let result = self.with_environment(program, wrappers, |vm| vm.run_main(program, scope)); + unwind_exit(result) + } + + /// What a program contributes to `global`, in place for the duration of `f`. + /// + /// Everything entering a program from outside needs this, not only its main + /// chunk: a function reached through [`call_fn`](Self::call_fn) can call + /// whatever the compiler left rhai to interpret, and rhai looks for it in + /// `global.lib`. + fn with_environment( + &mut self, + program: &Program, + wrappers: Option, + f: impl FnOnce(&mut Self) -> T, + ) -> T { let orig_source = mem::replace(&mut self.global.source, program.source().cloned()); let orig_lib_len = self.global.lib.len(); if let Some(lib) = program.lib() { @@ -587,6 +625,20 @@ impl<'e> Vm<'e> { program.resolver().cloned(), ); + let result = f(self); + + #[cfg(not(feature = "no_module"))] + { + self.global.embedded_module_resolver = orig_resolver; + } + self.global.lib.truncate(orig_lib_len); + self.global.source = orig_source; + + result + } + + /// The main chunk, against an environment the caller has already installed. + fn run_main(&mut self, program: &Program, scope: &mut Scope) -> VmResult { self.fault_pc = None; let mut pc = program.main().entry() as usize; // Slots are indices into the caller's scope, so a caller that arrives @@ -596,21 +648,7 @@ impl<'e> Vm<'e> { if result.is_err() { self.fault_pc = Some(pc); } - - #[cfg(not(feature = "no_module"))] - { - self.global.embedded_module_resolver = orig_resolver; - } - self.global.lib.truncate(orig_lib_len); - self.global.source = orig_source; - - // Rhai unwinds `return` and `exit` as errors rather than returning - // them; `eval_global_statements` is where they turn back into values, - // and anything entering a program from outside has to do the same. - result.or_else(|err| match *err { - EvalAltResult::Return(out, ..) | EvalAltResult::Exit(out, ..) => Ok(out), - _ => Err(err), - }) + result } fn pop(&mut self) -> Result> { diff --git a/tests/grain/call_fn.rs b/tests/grain/call_fn.rs new file mode 100644 index 000000000..33ec9e4f3 --- /dev/null +++ b/tests/grain/call_fn.rs @@ -0,0 +1,127 @@ +//! What a program contributes to `global` while [`Vm::call_fn`] runs. +//! +//! A program's library, source and module resolver used to be installed around +//! its *main chunk* only, so a function reached through `call_fn` ran without +//! them. That is not a corner: the compiler leaves anything it cannot lower as +//! an AST in the library, and rhai finds it only in `global.lib`. + +use rhai::grain::{Compiler, Vm}; +use rhai::{CallFnOptions, Dynamic, Engine, EvalAltResult, Scope, INT}; + +/// A receiver for the methods below. +fn holder(count: INT) -> Dynamic { + Dynamic::from_map( + [("count".into(), Dynamic::from(count))] + .into_iter() + .collect(), + ) +} + +/// `this` is what keeps `inner` an AST in the program's library, which is the +/// only way to observe from outside whether the library is installed. +const PROGRAM: &str = " + fn labelled() { this.count } + fn outer(m) { m.labelled() } +"; + +/// The same shape, but the un-lowered function's name is also a rhai built-in +/// (`Dynamic::tag`). Without the library the call does not fail — it silently +/// resolves to the built-in getter and answers 0. +const SHADOWED: &str = " + fn tag() { this.count } + fn outer(m) { m.tag() } +"; + +fn walked(engine: &Engine, source: &str) -> INT { + let ast = engine.compile(source).unwrap(); + engine + .call_fn(&mut Scope::new(), &ast, "outer", (holder(7),)) + .unwrap() +} + +fn run(engine: &Engine, source: &str) -> INT { + let ast = engine.compile(source).unwrap(); + let program = Compiler::new().compile(&ast); + Vm::new(engine) + .call_fn(&mut Scope::new(), &program, "outer", (holder(7),)) + .unwrap() +} + +#[test] +fn a_call_reaches_a_function_the_compiler_left_to_rhai() { + let engine = Engine::new(); + assert_eq!(walked(&engine, PROGRAM), 7); + assert_eq!(run(&engine, PROGRAM), walked(&engine, PROGRAM)); +} + +#[test] +fn a_missing_library_cannot_be_answered_by_a_builtin_of_the_same_name() { + let engine = Engine::new(); + assert_eq!(walked(&engine, SHADOWED), 7); + assert_eq!(run(&engine, SHADOWED), walked(&engine, SHADOWED)); +} + +/// The environment is now installed once around both halves of the call, so the +/// half that used to have it must not have lost it. +#[test] +fn the_main_chunk_still_runs_before_the_call() { + let engine = Engine::new(); + let ast = engine.compile("fn outer() { 1 } let started = 9;").unwrap(); + let program = Compiler::new().compile(&ast); + + let mut scope = Scope::new(); + let value: INT = Vm::new(&engine) + .call_fn_with_options( + CallFnOptions::new().rewind_scope(false), + &mut scope, + &program, + "outer", + (), + ) + .unwrap(); + + assert_eq!(value, 1); + // What the main chunk declared is left in the caller's scope, as + // `Engine::eval_ast_with_scope` would leave it. + assert_eq!(scope.get_value::("started"), Some(9)); +} + +#[test] +fn eval_ast_off_skips_the_main_chunk() { + let engine = Engine::new(); + let ast = engine.compile("fn outer() { 1 } let started = 9;").unwrap(); + let program = Compiler::new().compile(&ast); + + let mut scope = Scope::new(); + let _: INT = Vm::new(&engine) + .call_fn_with_options( + CallFnOptions::new().eval_ast(false).rewind_scope(false), + &mut scope, + &program, + "outer", + (), + ) + .unwrap(); + + assert!(scope.get_value::("started").is_none()); +} + +#[test] +fn an_error_inside_a_call_carries_the_program_source() { + let engine = Engine::new(); + let mut ast = engine.compile(r#"fn outer() { throw "boom" }"#).unwrap(); + ast.set_source("handlers.rhai"); + let program = Compiler::new().compile(&ast); + + let err = *Vm::new(&engine) + .call_fn::(&mut Scope::new(), &program, "outer", ()) + .unwrap_err(); + + match err { + EvalAltResult::ErrorInFunctionCall(name, source, ..) => { + assert_eq!(name, "outer"); + assert_eq!(source, "handlers.rhai"); + } + other => panic!("expected a wrapped call error, got {other:?}"), + } +} diff --git a/tests/mod.rs b/tests/mod.rs index 99648c8e2..1a1653e61 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -6,6 +6,7 @@ mod grain { // `allocation` is deliberately absent: it owns a counting global allocator // and is its own binary, declared in Cargo.toml. + mod call_fn; mod callback; mod differential; mod format; From f9b2b0539c6c8ba76d57e1d59121e93d09ae6681 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 12:33:44 -0700 Subject: [PATCH 16/33] silence must_use on the main chunk's discarded value Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/vm/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 2f7dc07f7..998ce5196 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -482,12 +482,12 @@ impl<'e> Vm<'e> { // the main chunk that may precede it: the function being called can // reach whatever the compiler left rhai to interpret, and rhai looks for // it in `global.lib`. - let mut evaluated = Ok(Dynamic::UNIT); + let mut evaluated = Ok(()); let result = self.with_environment(program, None, |vm| { if options.eval_ast { // Run for the scope it leaves behind; the body's own value is // not what the caller asked for. - evaluated = unwind_exit(vm.run_main(program, scope)); + evaluated = unwind_exit(vm.run_main(program, scope)).map(|_| ()); if evaluated.is_err() { return Ok(Dynamic::UNIT); } From 1d138b9f82e068efcc795117fcbaef96186e98f5 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 12:37:27 -0700 Subject: [PATCH 17/33] cargo fmt Co-Authored-By: Claude Opus 5 (1M context) --- tests/grain/call_fn.rs | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/tests/grain/call_fn.rs b/tests/grain/call_fn.rs index 33ec9e4f3..6e7bf7af1 100644 --- a/tests/grain/call_fn.rs +++ b/tests/grain/call_fn.rs @@ -10,11 +10,7 @@ use rhai::{CallFnOptions, Dynamic, Engine, EvalAltResult, Scope, INT}; /// A receiver for the methods below. fn holder(count: INT) -> Dynamic { - Dynamic::from_map( - [("count".into(), Dynamic::from(count))] - .into_iter() - .collect(), - ) + Dynamic::from_map([("count".into(), Dynamic::from(count))].into_iter().collect()) } /// `this` is what keeps `inner` an AST in the program's library, which is the @@ -34,17 +30,13 @@ const SHADOWED: &str = " fn walked(engine: &Engine, source: &str) -> INT { let ast = engine.compile(source).unwrap(); - engine - .call_fn(&mut Scope::new(), &ast, "outer", (holder(7),)) - .unwrap() + engine.call_fn(&mut Scope::new(), &ast, "outer", (holder(7),)).unwrap() } fn run(engine: &Engine, source: &str) -> INT { let ast = engine.compile(source).unwrap(); let program = Compiler::new().compile(&ast); - Vm::new(engine) - .call_fn(&mut Scope::new(), &program, "outer", (holder(7),)) - .unwrap() + Vm::new(engine).call_fn(&mut Scope::new(), &program, "outer", (holder(7),)).unwrap() } #[test] @@ -71,13 +63,7 @@ fn the_main_chunk_still_runs_before_the_call() { let mut scope = Scope::new(); let value: INT = Vm::new(&engine) - .call_fn_with_options( - CallFnOptions::new().rewind_scope(false), - &mut scope, - &program, - "outer", - (), - ) + .call_fn_with_options(CallFnOptions::new().rewind_scope(false), &mut scope, &program, "outer", ()) .unwrap(); assert_eq!(value, 1); @@ -94,13 +80,7 @@ fn eval_ast_off_skips_the_main_chunk() { let mut scope = Scope::new(); let _: INT = Vm::new(&engine) - .call_fn_with_options( - CallFnOptions::new().eval_ast(false).rewind_scope(false), - &mut scope, - &program, - "outer", - (), - ) + .call_fn_with_options(CallFnOptions::new().eval_ast(false).rewind_scope(false), &mut scope, &program, "outer", ()) .unwrap(); assert!(scope.get_value::("started").is_none()); @@ -113,9 +93,7 @@ fn an_error_inside_a_call_carries_the_program_source() { ast.set_source("handlers.rhai"); let program = Compiler::new().compile(&ast); - let err = *Vm::new(&engine) - .call_fn::(&mut Scope::new(), &program, "outer", ()) - .unwrap_err(); + let err = *Vm::new(&engine).call_fn::(&mut Scope::new(), &program, "outer", ()).unwrap_err(); match err { EvalAltResult::ErrorInFunctionCall(name, source, ..) => { From cd3c7da6577a2a7baba2dac836ccac7facbfde25 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 12:37:27 -0700 Subject: [PATCH 18/33] keep a chain's mutation when the walk raises after it The chain root is walked as a copy and written back afterwards, but the write-back was gated on the walk succeeding. Rhai reaches the entry through a live reference, so a step that mutates and then raises has already written: `let w = widget(1); try { w.bump_then_fail(); } catch(e) {} w.level` is 2 under the walker and was 1 here. Only that conjunct goes. A shared root is walked through its guard and still needs no write-back, and a read-only one still refused the mutation rather than making it. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/vm/mod.rs | 7 ++++++- tests/grain/corpus/mod.rs | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 998ce5196..d9faf7cc1 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -778,7 +778,12 @@ impl<'e> Vm<'e> { // Not a constant, which could not have been changed anyway: the walk // was handed a read-only value, so anything that would have mutated it // refused rather than mutating the copy. - if chain.mutates() && !shared && result.is_ok() && !root.is_read_only() { + // + // Whether the walk *failed* is not part of it. Rhai reaches the entry + // through a live `&mut`, so a step that mutates and then raises has + // already written — `a.push_then_fail()` inside a `try` leaves the + // push. Gating this on success would discard exactly that. + if chain.mutates() && !shared && !root.is_read_only() { match at { RootAt::Place(Site::Slot(index)) => *scope.get_mut_by_index(index) = root, RootAt::Place(Site::Name(name)) => { diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index 958d9b483..1922cae4e 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -72,7 +72,15 @@ pub fn engine() -> rhai::Engine { .register_fn("bump", |w: &mut Widget| w.level += 1) // Reads only. Whether rhai still writes back after one of these is // exactly the question this corpus is here to settle. - .register_fn("doubled", |w: &mut Widget| w.level * 2); + .register_fn("doubled", |w: &mut Widget| w.level * 2) + // Mutates and *then* fails. Rhai reaches a chain root through a live + // reference, so the mutation has already landed by the time the error + // propagates; nothing else here can tell a write-back that happens from + // one that is skipped because the walk raised. + .register_fn("bump_then_fail", |w: &mut Widget| -> Result<(), Box> { + w.level += 1; + Err("bump_then_fail".into()) + }); engine .register_type_with_name::("Holder") @@ -509,4 +517,10 @@ pub const CASES: &[Case] = &[ // whether a setter runs at all. case("host_temp_pure", "let h = holder(3); h.inner.doubled()"), case("error_host_index_bounds", "let w = widget(1); w[99]"), + // A step that mutates and then raises. The error is caught, so what is + // being compared is whether the mutation reached the variable — rhai's does, + // because it never walked a copy. + case("host_mutation_before_a_failure_survives", "let w = widget(1); try { w.bump_then_fail(); } catch(e) {} w.level"), + case("host_mutation_before_a_failure_survives_in_a_map", "let m = #{ w: widget(1) }; try { m.w.bump_then_fail(); } catch(e) {} m.w.level"), + case("host_mutation_before_a_failure_survives_in_an_array", "let a = [widget(1)]; try { a[0].bump_then_fail(); } catch(e) {} a[0].level"), ]; From eb0a06bd8a45d284e87060dc3a6967ce891ef008 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 12:44:29 -0700 Subject: [PATCH 19/33] add the instruction surface for a bound `this` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four opcodes, a chain root and a call receiver, none of which the compiler emits yet. `this` is not a scope entry and no slot addresses it, so it gets a register of its own and these are the only things that reach it. LoadThis flattens and LoadThisShared does not, the split LoadLocal/LoadShared already makes: rhai reads `this` unflattened but flattens at nearly every consumer. RequireThis exists because `this = v` checks boundness before evaluating v, unlike the variable arm. AssignThis carries no name because rhai's failure here has none either. Root::This groups with Local and Named rather than Temporary — a temporary would walk a copy and drop a mutation. root_tag is append-only, so no format version bump; the two runtime sites that need the register itself refuse for now rather than misread. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/bytecode/chain.rs | 18 +++++++- src/grain/bytecode/code.rs | 90 +++++++++++++++++++++++++++++++++--- src/grain/bytecode/op.rs | 62 +++++++++++++++++++++++++ src/grain/bytecode/verify.rs | 71 +++++++++++++++++++++++++--- src/grain/format/mod.rs | 1 + src/grain/format/read.rs | 3 ++ src/grain/format/write.rs | 4 ++ src/grain/vm/mod.rs | 21 +++++++++ 8 files changed, 255 insertions(+), 15 deletions(-) diff --git a/src/grain/bytecode/chain.rs b/src/grain/bytecode/chain.rs index aae026d72..ddc3c494f 100644 --- a/src/grain/bytecode/chain.rs +++ b/src/grain/bytecode/chain.rs @@ -140,6 +140,22 @@ pub enum Root { pos: rhai::Position, }, + /// The frame's receiver. + /// + /// Grouped with the two above rather than with [`Root::Temporary`], and the + /// distinction is the whole reason this variant exists: `this.push(1)` has + /// to mutate the caller's value, and a temporary would walk a copy and drop + /// the mutation silently. + /// + /// Carries its own position because the chain instruction's table entry is + /// the `.` or the `[`, while `ErrorUnboundThis` is reported against the + /// `this` (`eval/chaining.rs:519-527`) — two positions one instruction + /// cannot give. [`Root::Named`] carries one for the same reason. + This { + /// Where the `this` is in the source. + pos: rhai::Position, + }, + /// A value the instruction takes off the operand stack, pushed above the /// step operands. /// @@ -181,7 +197,7 @@ impl Chain { #[must_use] pub fn roots_on_stack(&self) -> bool { match self.root { - Root::Local { .. } | Root::Named { .. } => false, + Root::Local { .. } | Root::Named { .. } | Root::This { .. } => false, Root::Temporary => true, } } diff --git a/src/grain/bytecode/code.rs b/src/grain/bytecode/code.rs index 98e30e68a..ca7ee2b66 100644 --- a/src/grain/bytecode/code.rs +++ b/src/grain/bytecode/code.rs @@ -149,6 +149,18 @@ pub mod tag { pub const CALL_NAMED_REF: u8 = 0x37; /// [`Op::LoadSharedNamed`](super::Op::LoadSharedNamed). pub const LOAD_SHARED_NAMED: u8 = 0x38; + /// [`Op::LoadThis`](super::Op::LoadThis). + pub const LOAD_THIS: u8 = 0x39; + /// [`Op::LoadThisShared`](super::Op::LoadThisShared). + pub const LOAD_THIS_SHARED: u8 = 0x3a; + /// [`Op::RequireThis`](super::Op::RequireThis). + pub const REQUIRE_THIS: u8 = 0x3b; + /// [`Op::AssignThis`](super::Op::AssignThis) with a plain `=`. + pub const ASSIGN_THIS: u8 = 0x3c; + /// [`Op::AssignThis`](super::Op::AssignThis) through an operator. + pub const ASSIGN_THIS_OP: u8 = 0x3d; + /// [`Op::CallRef`](super::Op::CallRef) through [`Receiver::This`](super::Receiver::This). + pub const CALL_THIS_REF: u8 = 0x3e; } /// How wide each tag's instruction is, with 0 for the tags that are not one. @@ -193,6 +205,14 @@ static WIDTHS: [u8; 256] = { widths[tag::SHARE_NAMED as usize] = 3; widths[tag::LOAD_SHARED as usize] = 3; widths[tag::LOAD_SHARED_NAMED as usize] = 3; + + // `this` is a register, so none of these needs an operand to address it. + widths[tag::LOAD_THIS as usize] = 1; + widths[tag::LOAD_THIS_SHARED as usize] = 1; + widths[tag::REQUIRE_THIS as usize] = 1; + widths[tag::ASSIGN_THIS as usize] = 1; + widths[tag::ASSIGN_THIS_OP as usize] = 3; + widths[tag::CALL_THIS_REF as usize] = 4; widths[tag::MAKE_CLOSURE as usize] = 3; widths[tag::PUSH_HANDLER as usize] = 5; widths[tag::PUSH_HANDLER_VAR as usize] = 7; @@ -373,6 +393,19 @@ pub fn assemble(ops: &[Op]) -> Result<(Vec, Vec), AssembleError> { ); } + Op::LoadThis => code.push(tag::LOAD_THIS), + Op::LoadThisShared => code.push(tag::LOAD_THIS_SHARED), + Op::RequireThis => code.push(tag::REQUIRE_THIS), + Op::AssignThis { op: None } => code.push(tag::ASSIGN_THIS), + Op::AssignThis { + op: Some(assign_op), + } => { + code.push(tag::ASSIGN_THIS_OP); + code.extend_from_slice( + &small(*assign_op as usize, "op-assignments")?.to_le_bytes(), + ); + } + Op::LoadNamed(name) => { code.push(tag::LOAD_NAMED); code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); @@ -442,14 +475,21 @@ pub fn assemble(ops: &[Op]) -> Result<(Vec, Vec), AssembleError> { argc, receiver, } => { - let (tag, operand) = match receiver { - Receiver::Local(slot) => (tag::CALL_LOCAL_REF, *slot), - Receiver::Named(var) => (tag::CALL_NAMED_REF, small(*var as usize, "names")?), + // `this` is a register and needs no operand to address it, so + // its encoding is the other two minus the trailing `u16`. + let operand = match receiver { + Receiver::Local(slot) => Some((tag::CALL_LOCAL_REF, *slot)), + Receiver::Named(var) => { + Some((tag::CALL_NAMED_REF, small(*var as usize, "names")?)) + } + Receiver::This => None, }; - code.push(tag); + code.push(operand.map_or(tag::CALL_THIS_REF, |(tag, _)| tag)); code.extend_from_slice(&small(*name as usize, "names")?.to_le_bytes()); code.push(*argc); - code.extend_from_slice(&operand.to_le_bytes()); + if let Some((_, operand)) = operand { + code.extend_from_slice(&operand.to_le_bytes()); + } } Op::Rotate(under) => { @@ -641,6 +681,10 @@ fn encoded_width(op: &Op) -> usize { | Op::InterpolateEnd | Op::MakeFnPtr | Op::IsShared + | Op::LoadThis + | Op::LoadThisShared + | Op::RequireThis + | Op::AssignThis { op: None } | Op::Return => 1, Op::Curry(..) | Op::CallFnPtr { .. } | Op::Rotate(..) => 2, Op::Const(..) @@ -661,8 +705,13 @@ fn encoded_width(op: &Op) -> usize { | Op::MakeClosure(..) | Op::MakeArray(..) | Op::MakeMap(..) + | Op::AssignThis { op: Some(..) } | Op::CheckSize { .. } => 3, - Op::Call { op: None, .. } => 4, + Op::Call { op: None, .. } + | Op::CallRef { + receiver: Receiver::This, + .. + } => 4, Op::AssignLocal { op: None, .. } | Op::AssignNamed { op: Some(..), .. } | Op::Jump(..) @@ -676,7 +725,11 @@ fn encoded_width(op: &Op) -> usize { catch_var: Some(..), .. } => 7, - Op::Call { op: Some(..), .. } | Op::CallRef { .. } => 6, + Op::Call { op: Some(..), .. } + | Op::CallRef { + receiver: Receiver::Local(..) | Receiver::Named(..), + .. + } => 6, Op::AssignLocal { op: Some(..), .. } => 7, } } @@ -720,6 +773,14 @@ pub fn decode(code: &[u8], at: usize) -> Option { op: Some(u32::from(small(3)?)), }, + tag::LOAD_THIS => Op::LoadThis, + tag::LOAD_THIS_SHARED => Op::LoadThisShared, + tag::REQUIRE_THIS => Op::RequireThis, + tag::ASSIGN_THIS => Op::AssignThis { op: None }, + tag::ASSIGN_THIS_OP => Op::AssignThis { + op: Some(u32::from(small(1)?)), + }, + tag::DECLARE_LOCAL => Op::DeclareLocal { name: u32::from(small(1)?), is_const: false, @@ -755,6 +816,11 @@ pub fn decode(code: &[u8], at: usize) -> Option { argc: code[at + 3], receiver: Receiver::Local(small(4)?), }, + tag::CALL_THIS_REF => Op::CallRef { + name: u32::from(small(1)?), + argc: code[at + 3], + receiver: Receiver::This, + }, tag::CALL_NAMED_REF => Op::CallRef { name: u32::from(small(1)?), argc: code[at + 3], @@ -908,6 +974,16 @@ mod tests { argc: 2, receiver: Receiver::Named(5), }, + Op::CallRef { + name: 1, + argc: 2, + receiver: Receiver::This, + }, + Op::LoadThis, + Op::LoadThisShared, + Op::RequireThis, + Op::AssignThis { op: None }, + Op::AssignThis { op: Some(6) }, Op::Rotate(3), Op::UnwindTo(6), Op::Tick, diff --git a/src/grain/bytecode/op.rs b/src/grain/bytecode/op.rs index 006375a1c..4522cf39e 100644 --- a/src/grain/bytecode/op.rs +++ b/src/grain/bytecode/op.rs @@ -46,6 +46,24 @@ pub enum Receiver { /// only if a profile of a host-heavy script says so; a local, which is the /// common receiver by far, never makes one. Named(u32), + + /// The frame's receiver, for `f(this, ..)`. + /// + /// Rhai applies the same rewrite to `this` as to a variable, but only when + /// the receiver is neither shared nor curried (`func/call.rs:1409-1433`). + /// Shared-ness is a run-time property, so the value arrives on the stack as + /// argument zero and the call reaches for the register instead when it turns + /// out to be usable by reference — the deferral [`Receiver::Local`] already + /// makes for a read-only entry. + /// + /// Unlike either of the others, [`Op::LoadThis`] pushes it *before* the + /// remaining arguments. Rhai's two arms disagree about when `this` is read: + /// the by-reference one takes it after them (`func/call.rs:1417`), but the + /// fallback that a shared or unbound receiver lands in reads and flattens it + /// first (`:1462`). Reading first is what makes `f(this, { this = 9; 1 })` + /// pass the pre-mutation value, and an unbound `f(this, nosuch)` report + /// `ErrorUnboundThis` rather than `ErrorVariableNotFound`. + This, } /// One VM instruction, as the compiler emits it and a disassembly shows it. @@ -273,6 +291,50 @@ pub enum Op { /// that was missing. LoadSharedNamed(u32), + /// Push the receiver bound to the running frame. + /// + /// `this` is not a scope entry and no slot addresses it: rhai threads it + /// through evaluation as a parameter (`func/script.rs:29`) and keeps it out + /// of the `Scope` altogether. So it gets a register of its own, and these + /// four instructions are the only things that reach it. + /// + /// Flattens, as [`Op::LoadLocal`] does. Rhai reads `this` *unflattened* + /// (`eval/expr.rs:272`) but flattens at almost every consumer — a `let` + /// (`eval/stmt.rs:436`), an assignment's right-hand side (`:321`), a call's + /// arguments (`func/call.rs:1428`) — so the flattening read is the common + /// one and [`Op::LoadThisShared`] is the exception, exactly as it is for a + /// local. + /// + /// `ErrorUnboundThis` when the frame has no receiver. + LoadThis, + + /// The same without flattening. + /// + /// [`Op::LoadShared`]'s counterpart, for the three readers that have to see + /// the cell rather than what it holds: a `switch` subject, `is_shared`, and + /// a curried capture. + LoadThisShared, + + /// Raise `ErrorUnboundThis` if the frame has no receiver. Pushes nothing. + /// + /// `this = v` checks *before* it evaluates `v` (`eval/stmt.rs:299-302`), + /// unlike the variable arm, which evaluates the value first (`:319-323`). + /// Without a check of its own, `this = nosuch` in an unbound frame would + /// report `ErrorVariableNotFound` where rhai reports `ErrorUnboundThis`. + RequireThis, + + /// Pop a value and assign it to the frame's receiver, optionally through an + /// operator. + /// + /// [`Op::AssignLocal`] without the slot or the name, because `this` has + /// neither — and neither does rhai's own failure here: assigning to a + /// read-only receiver is `ErrorAssignmentToConstant("")` + /// (`eval/stmt.rs:118-122`), named for an expression that has no name. + AssignThis { + /// Index into the op-assignment pool; absent for a plain `=`. + op: Option, + }, + /// Pop a value and push whether it is a shared cell. /// /// Rhai answers `is_shared` before dispatch and registers no function for diff --git a/src/grain/bytecode/verify.rs b/src/grain/bytecode/verify.rs index 481d4862a..0b8b8a284 100644 --- a/src/grain/bytecode/verify.rs +++ b/src/grain/bytecode/verify.rs @@ -406,8 +406,13 @@ fn effect(op: &Op, pools: Pools) -> (usize, usize) { | Op::LoadShared(..) | Op::LoadSharedNamed(..) | Op::MakeClosure(..) + | Op::LoadThis + | Op::LoadThisShared | Op::EvalAst { .. } => (0, 1), + // A boundness check, which either raises or does nothing. + Op::RequireThis => (0, 0), + // Sharing is a change to the scope, not to the operand stack. Op::Share(..) | Op::ShareNamed(..) => (0, 0), @@ -415,7 +420,7 @@ fn effect(op: &Op, pools: Pools) -> (usize, usize) { // Pops the value, leaves nothing: the statement's unit value is a // separate `Op::Unit`. - Op::AssignLocal { .. } | Op::AssignNamed { .. } => (1, 0), + Op::AssignLocal { .. } | Op::AssignNamed { .. } | Op::AssignThis { .. } => (1, 0), Op::JumpIfFalse { .. } | Op::JumpIfTrue { .. } | Op::Switch(..) => (1, 0), @@ -429,12 +434,13 @@ fn effect(op: &Op, pools: Pools) -> (usize, usize) { // Arguments in, result out. Op::Call { argc, .. } => (*argc as usize, 1), - // A named receiver's value is argument zero like any other; a local's - // is not on the stack at all. An `argc` of zero names no receiver and - // is rejected when it runs. + // A named receiver's value is argument zero like any other, and so is + // `this` — which is pushed first rather than last, but the depth is the + // same either way. A local's is not on the stack at all. An `argc` of + // zero names no receiver and is rejected when it runs. Op::CallRef { argc, receiver, .. } => match receiver { Receiver::Local(..) => ((*argc as usize).saturating_sub(1), 1), - Receiver::Named(..) => (*argc as usize, 1), + Receiver::Named(..) | Receiver::This => (*argc as usize, 1), }, // Reorders what is already there, so the depth is unchanged — but it @@ -495,7 +501,9 @@ fn check_indices(at: usize, code: &[u8], pools: Pools) -> Result<(), VerifyError match code[at] { tag::CONST => bounded(index(1), "constant", pools.consts), tag::DECLARE_LOCAL | tag::DECLARE_CONST => bounded(index(1), "name", pools.names), - tag::CALL | tag::CALL_LOCAL_REF => bounded(index(1), "name", pools.names), + tag::CALL | tag::CALL_LOCAL_REF | tag::CALL_THIS_REF => { + bounded(index(1), "name", pools.names) + } // The function's, then the receiver variable's. The slot a local // receiver names is not a pool index and is checked against the scope // when it runs, as every other slot is. @@ -521,6 +529,10 @@ fn check_indices(at: usize, code: &[u8], pools: Pools) -> Result<(), VerifyError bounded(index(3), "name", pools.names)?; bounded(index(5), "op-assignment", pools.assign_ops) } + // `this` needs no name, so the operator is the whole of it — and + // omitting this would hand `program.assign_op` an unchecked index out + // of a corrupt artifact. + tag::ASSIGN_THIS_OP => bounded(index(1), "op-assignment", pools.assign_ops), tag::EVAL_AST | tag::EVAL_AST_KEEP => bounded(index(1), "fragment", pools.residuals), tag::CHAIN => { bounded(index(1), "chain", pools.chains.len())?; @@ -549,7 +561,9 @@ fn check_chain_indices(at: usize, chain: &Chain, pools: Pools) -> Result<(), Ver Root::Local { name, .. } | Root::Named { name, .. } => { bounded(name, "name", pools.names)?; } - Root::Temporary => {} + // Neither names anything in a pool: a temporary has no name at all, and + // `this` is a register rather than an entry. + Root::This { .. } | Root::Temporary => {} } for step in &chain.steps { @@ -612,6 +626,49 @@ mod tests { assert_eq!(check(vec![Op::Unit, Op::Return]), Ok(vec![1])); } + /// `this` is a register, so reading it costs a push and nothing else, and + /// assigning to it consumes one without leaving anything behind. + #[test] + fn the_this_register_is_reached_without_touching_the_scope() { + assert_eq!(check(vec![Op::LoadThis, Op::Return]), Ok(vec![1])); + assert_eq!(check(vec![Op::LoadThisShared, Op::Return]), Ok(vec![1])); + assert_eq!( + check(vec![Op::RequireThis, Op::Unit, Op::Return]), + Ok(vec![1]) + ); + assert_eq!( + check(vec![ + Op::RequireThis, + Op::Unit, + Op::AssignThis { op: None }, + Op::Unit, + Op::Return + ]), + Ok(vec![1]) + ); + } + + /// The operator is the whole of `ASSIGN_THIS_OP`'s payload, so an artifact + /// naming one the pool does not have has to be refused here — nothing + /// downstream re-checks it. + #[test] + fn rejects_an_op_assignment_to_this_that_the_pool_does_not_have() { + let ops = vec![ + Op::Unit, + Op::AssignThis { op: Some(0) }, + Op::Unit, + Op::Return, + ]; + assert_eq!( + check(ops), + Err(VerifyError::BadIndex { + at: 1, + what: "op-assignment", + index: 0, + }) + ); + } + /// A chain is one instruction over an unbounded record, so almost all of it /// is in the pool rather than in the code. Checking only the record's own /// index would leave the rest of the instruction unverified. diff --git a/src/grain/format/mod.rs b/src/grain/format/mod.rs index f8ba70b4c..37a449b6b 100644 --- a/src/grain/format/mod.rs +++ b/src/grain/format/mod.rs @@ -67,6 +67,7 @@ mod root_tag { pub const LOCAL: u8 = 0x01; pub const TEMPORARY: u8 = 0x02; pub const NAMED: u8 = 0x03; + pub const THIS: u8 = 0x04; } /// Chain-step tags. Append only. diff --git a/src/grain/format/read.rs b/src/grain/format/read.rs index 3f9452a8e..8bb301293 100644 --- a/src/grain/format/read.rs +++ b/src/grain/format/read.rs @@ -268,6 +268,9 @@ fn get_chain(cursor: &mut Cursor) -> Result { name: cursor.index()?, pos: get_position(cursor)?, }, + root_tag::THIS => Root::This { + pos: get_position(cursor)?, + }, root_tag::TEMPORARY => Root::Temporary, tag => { return Err(ReadError::UnknownTag { diff --git a/src/grain/format/write.rs b/src/grain/format/write.rs index ed4d46987..e6ed4dac5 100644 --- a/src/grain/format/write.rs +++ b/src/grain/format/write.rs @@ -196,6 +196,10 @@ fn put_chain_spec(out: &mut Vec, chain: &Chain) { put_uvarint(out, u64::from(name)); put_position(out, pos); } + Root::This { pos } => { + out.push(root_tag::THIS); + put_position(out, pos); + } Root::Temporary => out.push(root_tag::TEMPORARY), } put_uvarint(out, u64::from(chain.operands)); diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index d9faf7cc1..1ff71bbcb 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -165,9 +165,16 @@ enum Indexed { /// `None` for a temporary, which has no name to give — and neither error can /// reach one: nothing assigns through a temporary, and flattening it on the way /// in means there is no cell left to contend for. +/// +/// `this` *can* reach both and has no name either, so it answers with the empty +/// one rather than with nothing. That is rhai's own answer: `Expr::ThisPtr` +/// carries no name, so assigning through a read-only receiver is +/// `ErrorAssignmentToConstant("")` (`eval/stmt.rs:118-122`). Answering `None` +/// here would report a malformed chunk instead. fn root_name<'p>(program: &'p Program, chain: &Chain) -> Option<&'p str> { match chain.root { Root::Local { name, .. } | Root::Named { name, .. } => program.name(name), + Root::This { .. } => Some(""), Root::Temporary => None, } } @@ -846,6 +853,13 @@ impl<'e> Vm<'e> { }) } + // The frame's receiver. Encodable and verifiable ahead of the + // register that holds it, so an artifact carrying one is refused + // rather than misread. + Root::This { .. } => Err(malformed( + "a chain rooted at `this` needs a bound receiver".to_string(), + )), + // A name has a position of its own, and it wins: the lookup below // can fail, and rhai blames the variable rather than the chain. Root::Named { name, pos: var_pos } => { @@ -1814,6 +1828,13 @@ impl<'e> Vm<'e> { .ok_or_else(|| malformed(format!("no name {var}")))?; (Site::Name(name), argc) } + // Encodable and verifiable ahead of the register that holds it, so + // an artifact carrying one is refused rather than misread. + Receiver::This => { + return Err(malformed( + "a call through `this` needs a bound receiver".to_string(), + )) + } }; let first = self .stack From d16c61fa9c9ddcfad0f1bcef78f006a203600003 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 12:57:37 -0700 Subject: [PATCH 20/33] bind a receiver to a compiled frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the register the four opcodes reach, and makes call_fn_with_options honour options.this_ptr instead of ignoring it. The register is owned, not borrowed: the value a binder hands over always lives in the operand stack or a caller's Scope, and neither can lend a &mut across the &mut self call that runs the callee. So the binder moves it in and call_compiled_with_this hands it back. Two properties fall out with no conditional anywhere — a receiver is never inherited, because every ordinary call installs None; and a mutation survives an error, because the handback runs on both paths. Verified against hand-built chunks, since the compiler still refuses to emit any of this: the leak test fails if call_compiled passes the caller's receiver down. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/vm/mod.rs | 474 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 448 insertions(+), 26 deletions(-) diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 1ff71bbcb..f13150d28 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -287,9 +287,56 @@ pub struct Vm<'e> { /// top-level statement of a chunk, saved and restored per frame. See /// [`Vm::execute`]. unwind_floor: usize, + /// The receiver bound to the frame currently running. + /// + /// Owned rather than borrowed: the value a binder has to hand over always + /// lives in `stack` or in a caller's `Scope`, and neither can lend a `&mut` + /// across the `&mut self` call that runs the callee. So the binder moves it + /// in, the frame owns it, and [`Vm::call_compiled_with_this`] hands it back + /// for the binder to put where it came from. + /// + /// Saved and restored per *call* rather than per frame, which is what makes + /// `this` never inherited: a plain nested call passes `None` and gets the + /// caller's back on the way out, reproducing `func/call.rs:669` without a + /// conditional anywhere. + this: Option, fault_pc: Option, } +/// Take a receiver for a callee to own, and say whether it goes back. +/// +/// The rule is `chain_root`'s `walkable` (see [`Vm::chain_root`]) plus the +/// question of ownership. A read-only receiver is *cloned*, because `take` +/// would leave `UNIT` behind and strip the constness the caller's slot is +/// carrying; nothing is written back, since rhai refuses the mutation rather +/// than making it. Anything else is taken, as `call_compiled_body` already +/// takes a call's arguments — a receiver is not shared with anything the callee +/// can reach, and taking it saves a deep clone of an array or a map per call. +/// +/// A shared receiver is taken like any other: the taken value *is* the cell, so +/// `is_shared(this)` stays true inside the body and a write lands where every +/// other holder can see it. It still goes back, because taking it left `UNIT` +/// where the cell was. +fn bind_this(receiver: &mut Dynamic) -> (Dynamic, bool) { + if receiver.is_read_only() { + (receiver.clone().into_read_only(), false) + } else { + (mem::take(receiver), true) + } +} + +/// Put a receiver back where [`bind_this`] took it from. +/// +/// Unconditional on how the call ended. Rhai reaches `this` through a pointer +/// into the caller's storage, so a body that mutates and then raises has +/// already written; a binder that only restored on success would discard +/// exactly that. +fn unbind_this(receiver: &mut Dynamic, taken: Option, write_back: bool) { + if let (true, Some(value)) = (write_back, taken) { + *receiver = value; + } +} + /// A `try` region. struct Handler { target: usize, @@ -320,6 +367,7 @@ impl<'e> Vm<'e> { handlers: Vec::new(), sizes: Vec::new(), unwind_floor: 0, + this: None, fault_pc: None, } } @@ -355,6 +403,9 @@ impl<'e> Vm<'e> { handlers: Vec::new(), sizes: Vec::new(), unwind_floor: 0, + // A crossing carries no receiver: rhai binds one only where it + // dispatches a method, and this arrives through `call_fn_raw`. + this: None, fault_pc: None, } } @@ -403,11 +454,31 @@ impl<'e> Vm<'e> { level: usize, pos: Position, ) -> VmResult { + self.call_function_with_this(program, name, args, level, pos, None) + .0 + } + + /// The same, against a receiver the callee owns for the duration. + /// + /// The receiver comes back however the call ended, for the caller to put + /// where it took it from — see [`bind_this`] and [`unbind_this`]. + fn call_function_with_this( + &mut self, + program: &Program, + name: &str, + args: Vec, + level: usize, + pos: Position, + this: Option, + ) -> (VmResult, Option) { let Some(function) = program.function_named(name, args.len()) else { - return Err(Box::new(EvalAltResult::ErrorFunctionNotFound( - format!("{name} ({} args)", args.len()), - pos, - ))); + return ( + Err(Box::new(EvalAltResult::ErrorFunctionNotFound( + format!("{name} ({} args)", args.len()), + pos, + ))), + this, + ); }; let (params, chunk) = (function.params.clone(), function.chunk); @@ -417,11 +488,12 @@ impl<'e> Vm<'e> { self.stack.extend(args); let restore = mem::replace(&mut self.global.level, level); - let result = self.call_compiled(program, name, ¶ms, chunk, first, pos); + let (result, this) = + self.call_compiled_with_this(program, name, ¶ms, chunk, first, pos, this); self.global.level = restore; self.stack.truncate(first); - result + (result, this) } /// Call one compiled function by name, instead of running the whole @@ -456,13 +528,13 @@ impl<'e> Vm<'e> { /// * `rewind_scope` truncates the scope back afterwards. On by default. /// * `tag` sets the evaluation's custom state. /// - /// `this_ptr` is accepted and ignored: a compiled function never reads - /// `this`, because the compiler refuses to lower a body that does - /// (`compile/mod.rs`), so a bound `this` could not be observed. A body - /// that uses `this` is still an AST in the program's library and reached - /// through [`Engine::call_fn`](crate::Engine::call_fn) instead. - /// `in_all_namespaces` is likewise ignored — this looks only in the - /// program's own compiled functions. + /// `this_ptr` binds the callee's receiver, as it does in rhai. A write + /// through `this` lands in the pointer's own `Dynamic` — including when the + /// call goes on to fail, because rhai reaches `this` through the caller's + /// storage and a body that mutates and then raises has already written. + /// + /// `in_all_namespaces` is ignored: this looks only in the program's own + /// compiled functions. /// /// # Errors /// @@ -481,26 +553,40 @@ impl<'e> Vm<'e> { args.parse(&mut arg_values); let orig_scope_len = scope.len(); + let mut this_ptr = options.this_ptr; if let Some(tag) = options.tag { self.global.tag = tag; } + // The pointer is the host's and outlives the call, so unlike every + // other binder this one can hold it across the whole thing. + let bound = this_ptr.as_deref_mut().map(bind_this); + let (this, write_back) = bound.map_or((None, false), |(v, w)| (Some(v), w)); + // The program's environment stays installed for the *call*, not only for // the main chunk that may precede it: the function being called can // reach whatever the compiler left rhai to interpret, and rhai looks for // it in `global.lib`. let mut evaluated = Ok(()); - let result = self.with_environment(program, None, |vm| { + let (result, returned) = self.with_environment(program, None, |vm| { if options.eval_ast { // Run for the scope it leaves behind; the body's own value is - // not what the caller asked for. + // not what the caller asked for. The main chunk gets no + // receiver, as `eval_global_statements` does not either. evaluated = unwind_exit(vm.run_main(program, scope)).map(|_| ()); if evaluated.is_err() { - return Ok(Dynamic::UNIT); + return (Ok(Dynamic::UNIT), this); } } - vm.call_function(program, name, arg_values, 0, Position::NONE) + vm.call_function_with_this(program, name, arg_values, 0, Position::NONE, this) }); + + // Before the errors below, both of them: rhai's mutation through `this` + // survives a failed call, and survives one whose result is the wrong + // type just as much. + if let Some(slot) = this_ptr { + unbind_this(slot, returned, write_back); + } evaluated?; if options.rewind_scope { @@ -1913,10 +1999,6 @@ impl<'e> Vm<'e> { value } - /// Kept out of the dispatch loop. Inlined, it is enough extra code to - /// change register allocation across every other instruction — measured as - /// a uniform slowdown on benchmarks that call no functions at all. - #[inline(never)] fn call_compiled( &mut self, program: &Program, @@ -1926,12 +2008,49 @@ impl<'e> Vm<'e> { first: usize, pos: Position, ) -> VmResult { - self.engine.track_operation(&mut self.global, pos)?; + self.call_compiled_with_this(program, name, params, chunk, first, pos, None) + .0 + } - self.global.level += 1; - let result = self.call_compiled_body(program, name, params, chunk, first, pos); - self.global.level -= 1; - result + /// The same, against a receiver the callee owns for the duration. + /// + /// Hands the receiver back however the call ended, so a body that mutated + /// `this` and then raised still gives its binder something to write back — + /// which is what rhai's pointer into the caller's storage does for free. + /// + /// Every compiled call comes through here, and [`Vm::call_compiled`] is this + /// with no receiver. That is what makes `this` per-call rather than + /// inherited: an ordinary call installs `None` and gives the caller's back + /// on the way out, so a callee can never read the receiver of the frame that + /// called it (`func/call.rs:669`). + /// + /// Kept out of the dispatch loop. Inlined, it is enough extra code to change + /// register allocation across every other instruction — measured as a + /// uniform slowdown on benchmarks that call no functions at all. + #[inline(never)] + fn call_compiled_with_this( + &mut self, + program: &Program, + name: &str, + params: &[u32], + chunk: crate::grain::bytecode::Chunk, + first: usize, + pos: Position, + this: Option, + ) -> (VmResult, Option) { + let saved = mem::replace(&mut self.this, this); + + let result = match self.engine.track_operation(&mut self.global, pos) { + Ok(()) => { + self.global.level += 1; + let result = self.call_compiled_body(program, name, params, chunk, first, pos); + self.global.level -= 1; + result + } + Err(err) => Err(err), + }; + + (result, mem::replace(&mut self.this, saved)) } fn call_compiled_body( @@ -2537,6 +2656,73 @@ impl<'e> Vm<'e> { self.store(program, op, &mut target, rhs, pos())?; } + code::tag::LOAD_THIS | code::tag::LOAD_THIS_SHARED => { + let value = self + .this + .as_ref() + .ok_or_else(|| Box::new(EvalAltResult::ErrorUnboundThis(pos())))?; + // Rhai's read is `this_ptr.cloned()` and does not flatten + // (`eval/expr.rs:272`); its consumers do. Which tag this is + // is which consumer asked. + self.stack.push(if tag == code::tag::LOAD_THIS { + value.flatten_clone() + } else { + value.clone() + }); + } + + code::tag::REQUIRE_THIS => { + if self.this.is_none() { + return Err(Box::new(EvalAltResult::ErrorUnboundThis(pos()))); + } + } + + code::tag::ASSIGN_THIS | code::tag::ASSIGN_THIS_OP => { + let op = if tag == code::tag::ASSIGN_THIS_OP { + let index = u32::from(small(1)?); + Some( + program + .assign_op(index) + .ok_or_else(|| malformed(format!("no op-assignment {index}")))?, + ) + } else { + None + }; + + // Flattened before assigning, as everywhere else. + let rhs = self.pop()?.flatten(); + + // Taken out of the register rather than borrowed from it: + // `store` wants the whole `Vm`, and a write lock into the + // field could not outlive that borrow. Put back on both + // paths — rhai's mutation survives an error, and a frame + // that lost its receiver would answer `ErrorUnboundThis` to + // every read after this one. + let mut this = self + .this + .take() + .ok_or_else(|| Box::new(EvalAltResult::ErrorUnboundThis(pos())))?; + + let outcome = if this.is_read_only() { + // Named for an expression that has no name, which is + // what rhai reports too (`eval/stmt.rs:118-122`). + Err(Box::new(EvalAltResult::ErrorAssignmentToConstant( + String::new(), + pos(), + ))) + } else { + // Written through, not over: a shared receiver has to + // keep its cell, as a captured local does. + match place(&mut this, "", pos()) { + Ok(mut target) => self.store(program, op, &mut target, rhs, pos()), + Err(err) => Err(err), + } + }; + + self.this = Some(this); + outcome?; + } + code::tag::POP => { let _ = self.pop()?; } @@ -3037,3 +3223,239 @@ impl<'e> Vm<'e> { } } } + +/// The `this` register, reached by hand-built chunks. +/// +/// The compiler does not emit any of these yet — it still refuses a body that +/// mentions `this` — so this is the only thing that executes them until it does. +/// Worth having on its own account regardless: what a hand-made artifact can say +/// is exactly what a verifier-plus-VM has to survive. +#[cfg(test)] +mod tests { + use super::*; + use crate::grain::bytecode::{assemble, Chunk, Op, Positions, Strings}; + use crate::grain::program::{Function, Parts}; + use crate::{CallFnOptions, Engine, Scope, INT}; + + /// A program of nullary functions, named `f` upwards in the order given. + /// + /// The main chunk does nothing: everything here is entered through + /// [`Vm::call_fn_with_options`], which is the only thing that can bind a + /// receiver. + fn program_of(bodies: &[&[Op]], consts: Vec) -> Program<'static> { + const NAMES: [&str; 2] = ["f", "g"]; + + let mut all = vec![Op::Unit, Op::Return]; + let mut spans = Vec::new(); + for body in bodies { + let start = all.len(); + all.extend_from_slice(body); + spans.push(start..all.len()); + } + + // Assembling a prefix gives the byte offset that prefix ends at, which + // is what a chunk is measured in. + let end_of = |ops: usize| { + assemble(&all[..ops]) + .expect("the test ops must assemble") + .0 + .len() as u32 + }; + let (code, _) = assemble(&all).expect("the test ops must assemble"); + + let functions = spans + .iter() + .enumerate() + .map(|(index, span)| Function { + name: index as u32, + params: Vec::new(), + chunk: Chunk::new(end_of(span.start), end_of(span.end), 8), + }) + .collect(); + + Program::new( + code.into(), + Chunk::new(0, end_of(2), 8), + functions, + Parts { + positions: Positions::default(), + residuals: Vec::new(), + consts, + names: Strings::new(NAMES), + tokens: Vec::new(), + assign_ops: Vec::new(), + chains: Vec::new(), + switches: Vec::new(), + lib: None, + #[cfg(not(feature = "no_module"))] + resolver: None, + source: None, + }, + ) + } + + /// The one function `f`, for the cases that need no callee. + fn one(ops: &[Op], consts: Vec) -> Program<'static> { + program_of(&[ops], consts) + } + + fn call(program: &Program, this: Option<&mut Dynamic>) -> Result> { + let engine = Engine::new(); + let mut options = CallFnOptions::new().eval_ast(false); + options.this_ptr = this; + Vm::new(&engine).call_fn_with_options(options, &mut Scope::new(), program, "f", ()) + } + + #[test] + fn a_bound_receiver_is_what_load_this_pushes() { + let program = one(&[Op::LoadThis, Op::Return], Vec::new()); + let mut this = Dynamic::from(7 as INT); + assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); + } + + #[test] + fn reading_an_unbound_receiver_is_an_error() { + let program = one(&[Op::LoadThis, Op::Return], Vec::new()); + let err = *call(&program, None).unwrap_err(); + assert!( + matches!( + &err, + EvalAltResult::ErrorInFunctionCall(_, _, inner, _) + if matches!(**inner, EvalAltResult::ErrorUnboundThis(..)) + ), + "expected an unbound `this`, got {err:?}" + ); + } + + /// `this = v` checks boundness before evaluating `v`, which is the whole + /// reason [`Op::RequireThis`] is a separate instruction. + #[test] + fn assigning_to_an_unbound_receiver_is_caught_before_the_value_runs() { + let program = one(&[Op::RequireThis, Op::Unit, Op::Return], Vec::new()); + let err = *call(&program, None).unwrap_err(); + assert!( + matches!( + &err, + EvalAltResult::ErrorInFunctionCall(_, _, inner, _) + if matches!(**inner, EvalAltResult::ErrorUnboundThis(..)) + ), + "expected an unbound `this`, got {err:?}" + ); + } + + #[test] + fn a_write_through_this_reaches_the_hosts_value() { + let program = one( + &[ + Op::Const(0), + Op::AssignThis { op: None }, + Op::Unit, + Op::Return, + ], + vec![Dynamic::from(9 as INT)], + ); + let mut this = Dynamic::from(1 as INT); + assert!(call(&program, Some(&mut this)).is_ok()); + assert_eq!(this.as_int(), Ok(9)); + } + + /// Rhai reaches `this` through the caller's storage, so a body that mutates + /// and then raises has already written. + #[test] + fn a_write_through_this_survives_a_failure_after_it() { + let program = one( + &[ + Op::Const(0), + Op::AssignThis { op: None }, + Op::Const(1), + Op::Throw, + ], + vec![Dynamic::from(9 as INT), Dynamic::from("boom")], + ); + let mut this = Dynamic::from(1 as INT); + assert!(call(&program, Some(&mut this)).is_err()); + assert_eq!(this.as_int(), Ok(9)); + } + + /// A callee gets `None`, whatever its caller was holding + /// (`func/call.rs:669`). No conditional makes that true — every ordinary + /// call goes through `call_compiled`, which installs `None`. + /// + /// Worth testing at all because the failure is invisible: a register that + /// leaked would only show up in a callee that reads `this`, and reading a + /// value that happens to be there looks like success. + #[test] + fn a_receiver_is_not_inherited_by_a_callee() { + // `f` has a receiver and calls `g`, which reads one it was never given. + let program = program_of( + &[ + &[ + Op::Call { + name: 1, + argc: 0, + op: None, + }, + Op::Return, + ], + &[Op::LoadThis, Op::Return], + ], + Vec::new(), + ); + + let mut this = Dynamic::from(7 as INT); + let err = *call(&program, Some(&mut this)).unwrap_err(); + assert!( + format!("{err:?}").contains("ErrorUnboundThis"), + "expected `g` to have no receiver, got {err:?}" + ); + } + + /// And the caller still has its own afterwards. + #[test] + fn a_callees_frame_does_not_disturb_the_callers_receiver() { + let program = program_of( + &[ + &[ + Op::Call { + name: 1, + argc: 0, + op: None, + }, + Op::Pop, + Op::LoadThis, + Op::Return, + ], + &[Op::Unit, Op::Return], + ], + Vec::new(), + ); + + let mut this = Dynamic::from(7 as INT); + assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); + } + + #[test] + fn a_read_only_receiver_refuses_the_write_and_is_left_alone() { + let program = one( + &[ + Op::Const(0), + Op::AssignThis { op: None }, + Op::Unit, + Op::Return, + ], + vec![Dynamic::from(9 as INT)], + ); + let mut this = Dynamic::from(1 as INT).into_read_only(); + let err = *call(&program, Some(&mut this)).unwrap_err(); + assert!( + matches!( + &err, + EvalAltResult::ErrorInFunctionCall(_, _, inner, _) + // Named for an expression that has no name. + if matches!(&**inner, EvalAltResult::ErrorAssignmentToConstant(name, ..) if name.is_empty()) + ), + "expected a refused write to a constant, got {err:?}" + ); + assert_eq!(this.as_int(), Ok(1)); + } +} From 3c594594eca50db58f4ba546d22d6734b15aa062 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:00:21 -0700 Subject: [PATCH 21/33] walk a chain rooted at `this` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root::This resolves to the frame's register and writes back into it under the same rule a scope entry does — which is why it is not Root::Temporary: `this.push(1)` has to reach the caller's value rather than a copy. ErrorUnboundThis is reported against the `this`, not the `.` after it, which is what the position on the variant is for. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/vm/mod.rs | 105 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 9 deletions(-) diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index f13150d28..4fe3bd1d4 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -130,6 +130,9 @@ enum RootAt<'a> { /// A value with a name but no entry behind it — a resolver's answer, or a /// module's constant. Constant, + /// The frame's receiver, which is a register rather than a scope entry but + /// writes back for the same reason a [`RootAt::Place`] does. + This, /// A value with no name either: `[1, 2].len()`, `f().x`. Nothing can be /// assigned to one, because rhai's parser refuses it outright. Temporary, @@ -885,6 +888,14 @@ impl<'e> Vm<'e> { .ok_or_else(|| malformed(format!("`{name}` stopped being writable")))?; *entry = root; } + // The register, for the same reason and under the same rule: + // `this.push(1)` has to reach the caller's value, and the + // binder that owns it is what carries it back out of the frame. + RootAt::This => { + if let Some(entry) = self.this.as_mut() { + *entry = root; + } + } RootAt::Constant | RootAt::Temporary => {} } } @@ -939,12 +950,20 @@ impl<'e> Vm<'e> { }) } - // The frame's receiver. Encodable and verifiable ahead of the - // register that holds it, so an artifact carrying one is refused - // rather than misread. - Root::This { .. } => Err(malformed( - "a chain rooted at `this` needs a bound receiver".to_string(), - )), + // The `this` position wins over the chain's for the same reason a + // name's does: this lookup can fail, and rhai blames the `this` + // rather than the `.` after it (`eval/chaining.rs:519-527`). + Root::This { pos: this_pos } => { + let value = self + .this + .as_ref() + .ok_or_else(|| Box::new(EvalAltResult::ErrorUnboundThis(this_pos)))?; + Ok(ChainRoot { + at: RootAt::This, + value: walkable(value), + pos: this_pos, + }) + } // A name has a position of its own, and it wins: the lookup below // can fail, and rhai blames the variable rather than the chain. @@ -3233,7 +3252,7 @@ impl<'e> Vm<'e> { #[cfg(test)] mod tests { use super::*; - use crate::grain::bytecode::{assemble, Chunk, Op, Positions, Strings}; + use crate::grain::bytecode::{assemble, Chain, Chunk, Op, Positions, Step, Strings, Tail}; use crate::grain::program::{Function, Parts}; use crate::{CallFnOptions, Engine, Scope, INT}; @@ -3243,7 +3262,18 @@ mod tests { /// [`Vm::call_fn_with_options`], which is the only thing that can bind a /// receiver. fn program_of(bodies: &[&[Op]], consts: Vec) -> Program<'static> { - const NAMES: [&str; 2] = ["f", "g"]; + program_with_chains(bodies, consts, Vec::new()) + } + + /// The same, for the chain instruction, whose record lives in a pool rather + /// than in the code. Name indices are positions in `NAMES`. + fn program_with_chains( + bodies: &[&[Op]], + consts: Vec, + chains: Vec, + ) -> Program<'static> { + /// `f` and `g` are the functions; the rest are for chain steps to name. + const NAMES: [&str; 4] = ["f", "g", "push", "len"]; let mut all = vec![Op::Unit, Op::Return]; let mut spans = Vec::new(); @@ -3284,7 +3314,7 @@ mod tests { names: Strings::new(NAMES), tokens: Vec::new(), assign_ops: Vec::new(), - chains: Vec::new(), + chains, switches: Vec::new(), lib: None, #[cfg(not(feature = "no_module"))] @@ -3434,6 +3464,63 @@ mod tests { assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); } + /// A chain rooted at `this` mutates the caller's value rather than a copy. + /// That is the whole reason `Root::This` is not `Root::Temporary`. + #[test] + fn a_chain_rooted_at_this_mutates_the_hosts_value() { + let program = program_with_chains( + &[&[Op::Const(0), Op::Chain(0), Op::Return]], + vec![Dynamic::from(2 as INT)], + vec![Chain { + root: Root::This { + pos: Position::NONE, + }, + steps: vec![Step::Method { + name: 2, // `push` + argc: 1, + operand: 0, + pos: Position::NONE, + }], + tail: Tail::Read, + operands: 1, + }], + ); + + let mut this = Dynamic::from(vec![Dynamic::from(1 as INT)]); + assert!(call(&program, Some(&mut this)).is_ok()); + + let array = this.into_array().expect("still an array"); + let items: Vec = array.iter().map(|v| v.as_int().unwrap()).collect(); + assert_eq!(items, vec![1, 2]); + } + + #[test] + fn a_chain_rooted_at_an_unbound_this_is_an_error() { + let program = program_with_chains( + &[&[Op::Chain(0), Op::Return]], + Vec::new(), + vec![Chain { + root: Root::This { + pos: Position::NONE, + }, + steps: vec![Step::Method { + name: 3, // `len` + argc: 0, + operand: 0, + pos: Position::NONE, + }], + tail: Tail::Read, + operands: 0, + }], + ); + + let err = *call(&program, None).unwrap_err(); + assert!( + format!("{err:?}").contains("ErrorUnboundThis"), + "expected an unbound `this`, got {err:?}" + ); + } + #[test] fn a_read_only_receiver_refuses_the_write_and_is_left_alone() { let program = one( From 2903a13b836adfcc7bdd59d7502a7f29802bfb57 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:11:03 -0700 Subject: [PATCH 22/33] call with `this` as the first argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Receiver::This takes a path of its own rather than a third Site, because the register is not a scope entry. Rhai's rewrite fires for a receiver that is not shared, and read-only is deliberately not part of that test — unlike the variable arm, which copies a constant before deciding. The dispatch arm for CALL_THIS_REF was missing until a test found it: that site is one of the silent ones, where a new tag falls through to the wildcard and reports a malformed chunk rather than failing to compile. All six new tags are now covered at widths, decode and dispatch. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/vm/mod.rs | 155 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 143 insertions(+), 12 deletions(-) diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 4fe3bd1d4..be55f29f3 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -1917,6 +1917,12 @@ impl<'e> Vm<'e> { )); } + // The register is not a scope entry, so it takes a path of its own + // rather than a third [`Site`]. + if let Receiver::This = receiver { + return self.call_by_this(program, name_index, name, argc, pos); + } + // A named receiver's value is already argument zero — [`Op::LoadNamed`] // put it there. A local's is not on the stack at all. let (at, on_stack) = match receiver { @@ -1933,13 +1939,7 @@ impl<'e> Vm<'e> { .ok_or_else(|| malformed(format!("no name {var}")))?; (Site::Name(name), argc) } - // Encodable and verifiable ahead of the register that holds it, so - // an artifact carrying one is refused rather than misread. - Receiver::This => { - return Err(malformed( - "a call through `this` needs a bound receiver".to_string(), - )) - } + Receiver::This => unreachable!("taken above"), }; let first = self .stack @@ -2018,6 +2018,76 @@ impl<'e> Vm<'e> { value } + /// The same again, with `this` as the first argument. + /// + /// [`Op::LoadThis`] has already pushed a flattened snapshot as argument + /// zero — *before* the other arguments, unlike either of the other two + /// receivers, because the path a shared or unbound receiver takes reads + /// `this` first (`func/call.rs:1462`) where the by-reference path takes a + /// pointer to it afterwards (`:1417`). Reading first is what makes an + /// unbound `f(this, nosuch)` report `ErrorUnboundThis` rather than the + /// argument's failure. + /// + /// The snapshot is what gets passed when the register cannot be lent out, + /// and dead weight when it can — the trade [`Receiver::Named`] makes too. + #[inline(never)] + fn call_by_this( + &mut self, + program: &Program, + name_index: u32, + name: &str, + argc: usize, + pos: Position, + ) -> VmResult { + let first = self + .stack + .len() + .checked_sub(argc) + .ok_or_else(|| malformed("call with too few arguments".to_string()))?; + + // Rhai turns `f(this, ..)` into `this.f(..)` for a receiver that is not + // shared, and read-only is *not* part of that test — unlike the variable + // arm, which copies a constant before deciding (`func/call.rs:1449`). + // A function this compiler lowered copies its first argument whatever it + // is handed, exactly as rhai copies one before running a script function + // (`func/call.rs:661`), so a compiled callee rules a reference out too. + let by_reference = self.this.as_ref().map_or(false, |value| !is_shared!(value)) + && program.function(name_index, argc).is_none(); + + if !by_reference { + let value = self.call_stacked(program, name_index, argc, first, pos); + self.stack.truncate(first); + return value; + } + + let value = { + let entry = self + .this + .as_mut() + .ok_or_else(|| malformed("`this` stopped being bound".to_string()))?; + // Argument zero is the snapshot, dead now that there is a register + // to reach through. + let mut args: Vec<&mut Dynamic> = core::iter::once(entry) + .chain(self.stack[first + 1..].iter_mut()) + .collect(); + // A scope of the callee's own, for [`Vm::call_stacked`]'s reason. + let mut detached = Scope::new(); + let mut context = EvalContext::new( + self.engine, + &mut self.global, + &mut self.caches, + &mut detached, + None, + ); + context + .call_fn_raw(name, true, false, &mut args) + .map_err(|err| dispatch_failure(err, pos)) + }; + + self.stack.truncate(first); + value + } + fn call_compiled( &mut self, program: &Program, @@ -2855,13 +2925,17 @@ impl<'e> Vm<'e> { self.stack.push(value); } - code::tag::CALL_LOCAL_REF | code::tag::CALL_NAMED_REF => { + code::tag::CALL_LOCAL_REF + | code::tag::CALL_NAMED_REF + | code::tag::CALL_THIS_REF => { let name_index = u32::from(small(1)?); let argc = code[pc + 3] as usize; - let receiver = if tag == code::tag::CALL_LOCAL_REF { - Receiver::Local(small(4)?) - } else { - Receiver::Named(u32::from(small(4)?)) + // `this` is a register, so this one carries no operand for + // the receiver and is two bytes shorter. + let receiver = match tag { + code::tag::CALL_LOCAL_REF => Receiver::Local(small(4)?), + code::tag::CALL_NAMED_REF => Receiver::Named(u32::from(small(4)?)), + _ => Receiver::This, }; let value = self.call_by_reference( @@ -3494,6 +3568,63 @@ mod tests { assert_eq!(items, vec![1, 2]); } + /// `f(this, ..)` is rhai's method-call rewrite, so the receiver goes by + /// reference and a mutating native reaches the caller's value. + #[test] + fn this_as_a_first_argument_goes_by_reference() { + let program = one( + &[ + Op::LoadThis, + Op::Const(0), + Op::CallRef { + name: 2, // `push` + argc: 2, + receiver: Receiver::This, + }, + Op::Return, + ], + vec![Dynamic::from(2 as INT)], + ); + + let mut this = Dynamic::from(vec![Dynamic::from(1 as INT)]); + if let Err(err) = call(&program, Some(&mut this)) { + panic!("expected the push to succeed, got {err:?}"); + } + + let items: Vec = this + .into_array() + .expect("still an array") + .iter() + .map(|v| v.as_int().unwrap()) + .collect(); + assert_eq!(items, vec![1, 2]); + } + + /// The snapshot is pushed before the other arguments, so an unbound + /// receiver is what fails — not whatever the arguments would have done. + #[test] + fn an_unbound_this_beats_a_failing_argument() { + let program = one( + &[ + Op::LoadThis, + Op::LoadNamed(3), // `len`, which is no variable + Op::CallRef { + name: 2, + argc: 2, + receiver: Receiver::This, + }, + Op::Return, + ], + Vec::new(), + ); + + let err = *call(&program, None).unwrap_err(); + assert!( + format!("{err:?}").contains("ErrorUnboundThis"), + "expected the receiver to fail first, got {err:?}" + ); + } + #[test] fn a_chain_rooted_at_an_unbound_this_is_an_error() { let program = program_with_chains( From f79552016324c7989f350204b87770acc812cafa Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:12:48 -0700 Subject: [PATCH 23/33] hand the frame's receiver to a residual fragment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A body that uses `this` can still hold a fragment the compiler could not lower — `this?.x`, or an `import` inside one — and the walker was being handed None, so a `this` the surrounding instructions read perfectly well came back as ErrorUnboundThis. It goes by reference, so a write from the fragment lands in the same register. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/vm/mod.rs | 54 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index be55f29f3..5cbe23e2d 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -2828,20 +2828,28 @@ impl<'e> Vm<'e> { // is the same two calls behind a shim that only exists // under `custom_syntax`. Total language coverage rests on // this, so it must not depend on a feature. + // + // The frame's receiver goes with it, by reference. A body + // that uses `this` can still hold a fragment — `this?.x`, + // or a `this` body containing an `import` — and the walker + // has to read and write the same receiver the surrounding + // instructions do. The engine is copied out first so the + // four borrows below are of disjoint fields. + let engine = self.engine; let value = match expr { - Expr::Stmt(block) => self.engine.eval_stmt_block( + Expr::Stmt(block) => engine.eval_stmt_block( &mut self.global, &mut self.caches, scope, - None, + self.this.as_mut(), block.statements(), rewind_scope, ), - expr => self.engine.eval_expr( + expr => engine.eval_expr( &mut self.global, &mut self.caches, scope, - None, + self.this.as_mut(), expr, ), }?; @@ -3336,15 +3344,25 @@ mod tests { /// [`Vm::call_fn_with_options`], which is the only thing that can bind a /// receiver. fn program_of(bodies: &[&[Op]], consts: Vec) -> Program<'static> { - program_with_chains(bodies, consts, Vec::new()) + program_with(bodies, consts, Vec::new(), Vec::new()) } /// The same, for the chain instruction, whose record lives in a pool rather - /// than in the code. Name indices are positions in `NAMES`. + /// than in the code. fn program_with_chains( bodies: &[&[Op]], consts: Vec, chains: Vec, + ) -> Program<'static> { + program_with(bodies, consts, chains, Vec::new()) + } + + /// The general form. Name indices are positions in `NAMES`. + fn program_with( + bodies: &[&[Op]], + consts: Vec, + chains: Vec, + residuals: Vec, ) -> Program<'static> { /// `f` and `g` are the functions; the rest are for chain steps to name. const NAMES: [&str; 4] = ["f", "g", "push", "len"]; @@ -3383,7 +3401,7 @@ mod tests { functions, Parts { positions: Positions::default(), - residuals: Vec::new(), + residuals, consts, names: Strings::new(NAMES), tokens: Vec::new(), @@ -3538,6 +3556,28 @@ mod tests { assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); } + /// A fragment the compiler could not lower still sees the receiver. Without + /// this the walker would be handed `None` and report `ErrorUnboundThis` for + /// a `this` the surrounding instructions can read perfectly well. + #[test] + fn a_residual_fragment_reads_the_frames_receiver() { + let program = program_with( + &[&[ + Op::EvalAst { + residual: 0, + rewind_scope: false, + }, + Op::Return, + ]], + Vec::new(), + Vec::new(), + vec![Expr::ThisPtr(Position::NONE)], + ); + + let mut this = Dynamic::from(7 as INT); + assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); + } + /// A chain rooted at `this` mutates the caller's value rather than a copy. /// That is the whole reason `Root::This` is not `Root::Temporary`. #[test] From 6cda7b77310d57ccffe4c73a3871e8337dfef12e Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:20:10 -0700 Subject: [PATCH 24/33] resolve a method call against the compiled table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step::Method always went to rhai's dispatch, which finds a script function only in global.lib — so a `this` user that leaves the library would become unreachable the moment the compiler stops skipping it. It now tries our own table first, binding the receiver as the callee's `this`. Consulting the table first is safe: import pushes onto global.modules, not global.lib (eval/stmt.rs:947), and across the crate global.lib is only ever pushed where an AST is being run. That was the one ordering question the plan refused to guess at. Function carries this_type, and Program::method reproduces rhai's typed-hash-first, untyped-fallback selection without hashing. A typed method stays invisible to function-call style, as it is in rhai. The field sits inside a positional record, so VERSION moves 6 -> 7 and the golden fixture is regenerated — two bytes wider, one varint per script function. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/compile/mod.rs | 8 ++ src/grain/format/mod.rs | 2 +- src/grain/format/read.rs | 6 ++ src/grain/format/write.rs | 5 + src/grain/program.rs | 151 ++++++++++++++++++++++++++++++- src/grain/vm/mod.rs | 111 ++++++++++++++++++++++- tests/grain/fixtures/golden.rgrn | Bin 1759 -> 1761 bytes 7 files changed, 275 insertions(+), 8 deletions(-) diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index bc2f5d85f..ecdd2a9bf 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -154,6 +154,7 @@ impl Compiler { .map(|f| Function { name: f.name, params: f.params, + this_type: f.this_type, chunk: Chunk::new( offsets[f.first_op], offsets[f.first_op + f.op_count], @@ -242,6 +243,9 @@ enum Entry { struct LoweredFn { name: u32, params: Vec, + /// The declared receiver type, as a name-pool index. See + /// [`Function::this_type`](crate::grain::program::Function::this_type). + this_type: Option, first_op: usize, op_count: usize, } @@ -709,6 +713,10 @@ impl Lowering { Some(LoweredFn { name: self.push_name(def.name.clone()), params, + this_type: def + .this_type + .as_ref() + .map(|typed| self.push_name(typed.clone())), first_op, op_count: self.code.len() - first_op, }) diff --git a/src/grain/format/mod.rs b/src/grain/format/mod.rs index 37a449b6b..aaff1ec3c 100644 --- a/src/grain/format/mod.rs +++ b/src/grain/format/mod.rs @@ -60,7 +60,7 @@ const MAGIC: [u8; 4] = *b"RGRN"; /// Bumped when an encoding changes in a way an older reader would misread. /// Additive changes that an older reader would reject anyway — a new op tag, /// a new constant tag — do not need it. -const VERSION: u16 = 6; +const VERSION: u16 = 7; /// Where a chain starts. Append only. mod root_tag { diff --git a/src/grain/format/read.rs b/src/grain/format/read.rs index 8bb301293..1b8862ea5 100644 --- a/src/grain/format/read.rs +++ b/src/grain/format/read.rs @@ -185,12 +185,18 @@ pub(super) fn read(bytes: &[u8]) -> Result, ReadError> { let mut functions = Vec::new(); for _ in 0..cursor.uvarint()? { let name = cursor.index()?; + // Zero is "untyped"; anything else is an index one higher. + let this_type = match cursor.uvarint()? { + 0 => None, + raw => Some(u32::try_from(raw - 1).map_err(|_| ReadError::Truncated)?), + }; let mut params = Vec::new(); for _ in 0..cursor.uvarint()? { params.push(cursor.index()?); } functions.push(Function { name, + this_type, params, chunk: get_chunk(&mut cursor)?, }); diff --git a/src/grain/format/write.rs b/src/grain/format/write.rs index e6ed4dac5..41762ac6a 100644 --- a/src/grain/format/write.rs +++ b/src/grain/format/write.rs @@ -150,6 +150,11 @@ pub(super) fn write(program: &Program, positions: Positions) -> Result, put_uvarint(&mut out, program.functions().len() as u64); for function in program.functions() { put_uvarint(&mut out, u64::from(function.name)); + // Zero is "untyped", so an index arrives one higher. The field is why + // `VERSION` moved to 7: it sits inside a positional record, and a reader + // that did not expect it would take it for the parameter count and lose + // its place in every section that follows. + put_uvarint(&mut out, function.this_type.map_or(0, |t| u64::from(t) + 1)); put_uvarint(&mut out, function.params.len() as u64); for param in &function.params { put_uvarint(&mut out, u64::from(*param)); diff --git a/src/grain/program.rs b/src/grain/program.rs index aed1c749f..f4510c3a8 100644 --- a/src/grain/program.rs +++ b/src/grain/program.rs @@ -34,6 +34,17 @@ pub struct Function { /// Parameter names, in order, as name-pool indices. They become the /// callee's first locals, which is what makes them slot 0 upwards. pub params: Vec, + /// The receiver type this function was declared for, as a name-pool index. + /// + /// `fn .name()`. Rhai folds it into the function's hash rather than + /// checking it (`func/hashing.rs:159`), and tries the typed hash before the + /// plain one on a method call (`func/call.rs:614-629`) — so a typed function + /// and an untyped one of the same name and arity can both exist, and which + /// runs depends on the receiver's runtime type name. The string is what the + /// parser interned, which is already `Engine::map_type_name`'s answer. + /// + /// `None` for an ordinary function, which is nearly all of them. + pub this_type: Option, pub chunk: Chunk, } @@ -72,6 +83,13 @@ pub struct Program<'a> { /// might call it back. See [`Program::makes_fn_pointers`]. makes_fn_pointers: bool, + /// Whether any function was declared for a receiver type. + /// + /// Derived, not stored: [`Program::method`] is a linear scan on every method + /// call, and typed-first selection would double it for the overwhelming + /// majority of programs that have nothing typed to find. + has_typed_methods: bool, + /// Where each instruction came from, or [`Positions::Stripped`]. /// /// Separable on purpose: a device is shipped the code and the host keeps @@ -234,12 +252,14 @@ impl<'a> Program<'a> { parts: Parts<'a>, ) -> Self { let makes_fn_pointers = makes_fn_pointers(&code); + let has_typed_methods = functions.iter().any(|f| f.this_type.is_some()); let mut program = Self { code, main, functions, max_stack: 0, makes_fn_pointers, + has_typed_methods, positions: parts.positions, residuals: parts.residuals, consts: parts.consts, @@ -269,6 +289,7 @@ impl<'a> Program<'a> { functions: self.functions, max_stack: self.max_stack, makes_fn_pointers: self.makes_fn_pointers, + has_typed_methods: self.has_typed_methods, positions: self.positions, residuals: self.residuals, consts: self.consts, @@ -361,10 +382,46 @@ impl<'a> Program<'a> { /// Name and arity only, matching how rhai keys script functions. The name /// is an index into the pool the call site also indexes, so equal names /// have equal indices and this is two integer comparisons. + /// + /// Typed methods are invisible here. Rhai only ever tries a typed hash on a + /// *method* call (`func/call.rs:614`), so `fn .foo()` cannot be reached + /// as `foo()` — see [`Program::method`], which is the other door. pub(crate) fn function(&self, name: u32, argc: usize) -> Option<&Function> { self.functions .iter() - .find(|f| f.name == name && f.params.len() == argc) + .find(|f| f.name == name && f.params.len() == argc && f.this_type.is_none()) + } + + /// The compiled function a *method* call resolves to. + /// + /// `argc` excludes the receiver: `x.foo(1)` looks for the script function + /// `foo` of arity **one** and binds `this` to `x`, which is what the parser + /// hashes (`parser.rs:2128-2145`). That is the whole difference from + /// [`Program::function`], whose `argc` counts the receiver because the + /// rewrite it serves is function-call style. + /// + /// `typed` is the receiver's mapped type name. A function declared for it + /// wins, and an untyped one of the same name and arity is the fallback — + /// rhai's order, minus the hashing (`func/call.rs:614-629`). + pub(crate) fn method(&self, name: u32, argc: usize, typed: &str) -> Option<&Function> { + let matching = |f: &&Function| f.name == name && f.params.len() == argc; + + // Nearly every program has no typed method at all, and this is a linear + // scan on every method call — so the extra pass is bought only where + // there is something for it to find. + if self.has_typed_methods { + let found = self + .functions + .iter() + .find(|f| matching(f) && f.this_type.and_then(|t| self.name(t)) == Some(typed)); + if found.is_some() { + return found; + } + } + + self.functions + .iter() + .find(|f| matching(f) && f.this_type.is_none()) } /// The compiled function a *pointer* resolves to. @@ -373,9 +430,9 @@ impl<'a> Program<'a> { /// it may have been built from one at run time. A linear scan, which at /// these sizes beats a map and keeps the common indexed lookup untouched. pub(crate) fn function_named(&self, name: &str, argc: usize) -> Option<&Function> { - self.functions - .iter() - .find(|f| f.params.len() == argc && self.name(f.name) == Some(name)) + self.functions.iter().find(|f| { + f.params.len() == argc && f.this_type.is_none() && self.name(f.name) == Some(name) + }) } /// Whether this program can hand a function pointer to something that @@ -596,3 +653,89 @@ impl<'a> Program<'a> { self.source.as_ref() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::grain::bytecode::{assemble, Op, Positions, Strings}; + + /// Names: 0 `f`, 1 `i64`, 2 `string`. + fn program_of(functions: &[(u32, Option, usize)]) -> Program<'static> { + // Every chunk is the same two instructions; only the table matters here. + let (code, _) = assemble(&[Op::Unit, Op::Return]).expect("must assemble"); + let whole = Chunk::new(0, code.len() as u32, 8); + + let functions = functions + .iter() + .map(|&(name, this_type, argc)| Function { + name, + params: vec![0; argc], + this_type, + chunk: whole, + }) + .collect(); + + Program::new( + code.into(), + whole, + functions, + Parts { + positions: Positions::default(), + residuals: Vec::new(), + consts: Vec::new(), + names: Strings::new(["f", "i64", "string"]), + tokens: Vec::new(), + assign_ops: Vec::new(), + chains: Vec::new(), + switches: Vec::new(), + lib: None, + #[cfg(not(feature = "no_module"))] + resolver: None, + source: None, + }, + ) + } + + /// Rhai tries the receiver's type first and falls back to the untyped + /// function of the same name and arity (`func/call.rs:614-629`). + #[test] + fn a_typed_method_wins_over_an_untyped_one_of_the_same_arity() { + let program = program_of(&[(0, Some(1), 0), (0, None, 0)]); + + assert_eq!(program.method(0, 0, "i64").unwrap().this_type, Some(1)); + // No function declared for a string, so the untyped one answers. + assert_eq!(program.method(0, 0, "string").unwrap().this_type, None); + } + + /// A typed method is only ever reached through a method call: rhai computes + /// the typed hash nowhere else, so `foo()` cannot find `fn .foo()`. + #[test] + fn a_typed_method_is_unreachable_in_call_style() { + let program = program_of(&[(0, Some(1), 0)]); + + assert!(program.function(0, 0).is_none()); + assert!(program.function_named("f", 0).is_none()); + assert!(program.method(0, 0, "i64").is_some()); + } + + #[test] + fn arity_is_matched_before_the_receiver_type() { + let program = program_of(&[(0, Some(1), 1), (0, None, 0)]); + + // The typed one takes an argument, so a no-argument call is the untyped. + assert_eq!(program.method(0, 0, "i64").unwrap().this_type, None); + assert_eq!(program.method(0, 1, "i64").unwrap().this_type, Some(1)); + } + + #[test] + fn a_receiver_type_survives_the_round_trip() { + let program = program_of(&[(0, Some(1), 0), (0, None, 0)]); + + let bytes = program.write().expect("must be writable"); + let reloaded = Program::read(&bytes).expect("must load"); + + let typed: Vec<_> = reloaded.functions().iter().map(|f| f.this_type).collect(); + assert_eq!(typed, vec![Some(1), None]); + assert_eq!(reloaded.method(0, 0, "i64").unwrap().this_type, Some(1)); + } +} diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 5cbe23e2d..f2f306ee3 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -1085,17 +1085,56 @@ impl<'e> Vm<'e> { pos: step_pos, } => { let step_pos = *step_pos; + let name_index = *name; let name = program - .name(*name) - .ok_or_else(|| malformed(format!("no name {name}")))?; + .name(name_index) + .ok_or_else(|| malformed(format!("no name {name_index}")))?; let first = *operand as usize; let argc = *argc as usize; if first + argc > operands.len() { return Err(malformed("chain method arguments missing".to_string())); } + // A method call is where rhai tries the receiver's type before + // the plain name (`func/call.rs:614-629`), and the only place it + // does. `argc` already excludes the receiver, which is the arity + // the script side is keyed on (`parser.rs:2128-2145`). + // + // Consulting our own table first is safe because nothing can get + // between: `import` pushes onto `global.modules`, not + // `global.lib` (`eval/stmt.rs:947`), and `global.lib` is only + // ever pushed where an AST is being run. + let type_name = target.type_name(); + let compiled = { + let typed = self.engine.map_type_name(type_name); + program + .method(name_index, argc, typed) + .map(|f| (f.params.clone(), f.chunk)) + }; + let mut args: Vec = operands[first..first + argc].to_vec(); - let out = { + let out = if let Some((params, chunk)) = compiled { + // The receiver is moved into the frame and moved back, so a + // write through `this` lands here and the chain's own + // write-back carries it the rest of the way. + let (bound, write_back) = bind_this(target); + let at = self.stack.len(); + self.stack.extend(args); + let (result, returned) = self.call_compiled_with_this( + program, + name, + ¶ms, + chunk, + at, + step_pos, + Some(bound), + ); + self.stack.truncate(at); + // Before `?`: a body that mutated and then raised has + // already written, as it would through rhai's pointer. + unbind_this(target, returned, write_back); + result? + } else { let mut call_args: Vec<&mut Dynamic> = core::iter::once(&mut *target) .chain(args.iter_mut()) .collect(); @@ -3391,6 +3430,7 @@ mod tests { .map(|(index, span)| Function { name: index as u32, params: Vec::new(), + this_type: None, chunk: Chunk::new(end_of(span.start), end_of(span.end), 8), }) .collect(); @@ -3556,6 +3596,71 @@ mod tests { assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); } + /// A chain rooted at `this` whose method is a chunk of ours: the receiver + /// becomes the callee's `this`, which is the binding rhai does at + /// `func/call.rs:649-655` and the only place a method call differs from a + /// plain one. + #[test] + fn a_method_step_reaching_a_chunk_binds_the_receiver() { + let program = program_with_chains( + // `f` is `this.g()`; `g` is `this`. + &[&[Op::Chain(0), Op::Return], &[Op::LoadThis, Op::Return]], + Vec::new(), + vec![Chain { + root: Root::This { + pos: Position::NONE, + }, + steps: vec![Step::Method { + name: 1, // `g` + argc: 0, + operand: 0, + pos: Position::NONE, + }], + tail: Tail::Read, + operands: 0, + }], + ); + + let mut this = Dynamic::from(7 as INT); + assert_eq!(call(&program, Some(&mut this)).unwrap().as_int(), Ok(7)); + } + + /// And a write inside that callee travels back out through both frames: the + /// callee's register, the chain's root write-back, then the host's pointer. + #[test] + fn a_write_inside_a_method_step_reaches_the_host() { + let program = program_with_chains( + // `f` is `this.g()`; `g` is `this = 9`. + &[ + &[Op::Chain(0), Op::Return], + &[ + Op::Const(0), + Op::AssignThis { op: None }, + Op::Unit, + Op::Return, + ], + ], + vec![Dynamic::from(9 as INT)], + vec![Chain { + root: Root::This { + pos: Position::NONE, + }, + steps: vec![Step::Method { + name: 1, // `g` + argc: 0, + operand: 0, + pos: Position::NONE, + }], + tail: Tail::Read, + operands: 0, + }], + ); + + let mut this = Dynamic::from(1 as INT); + assert!(call(&program, Some(&mut this)).is_ok()); + assert_eq!(this.as_int(), Ok(9)); + } + /// A fragment the compiler could not lower still sees the receiver. Without /// this the walker would be handed `None` and report `ErrorUnboundThis` for /// a `this` the surrounding instructions can read perfectly well. diff --git a/tests/grain/fixtures/golden.rgrn b/tests/grain/fixtures/golden.rgrn index d71f299a38cd87ae0309d3972a2b5e7035f396de..b54feead3f7e13800e917f0ff22c9418934b6480 100644 GIT binary patch delta 51 zcmcc5`;eD4$UVr9eIn~50Slv~v{V!GBx6eh!_+jxw2jN>F>y06aWmdvdCJ1X!LZqn Hc_K3ajx|+Gu~i%%EH94Ie>X0 FGXR2{4&49% From 6092b07610e2f20bbf8b64742660877905a14478 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:26:24 -0700 Subject: [PATCH 25/33] lower a function body that uses `this` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refusal at compile/mod.rs:673 goes, and with it the reason a program with an event handler in it could not be written as an artifact at all — one such function kept rhai's whole function library alive, and format/write.rs refuses to write a program that still has one. Six arms: a bare read, the unflattened read the three cell-observing readers need, `this = v` ahead of the two variable arms as rhai's parser puts it, Root::This, Receiver::This, and LoadThis emitted before the other arguments rather than after. `this.call(f)` and `this.curry(f)` come for free through the existing arm. 22 corpus cases, which the differential harness compares against the walker on value, error variant, error position and leftover scope, and which the residual census requires to lower whole. uses_this is deleted: nothing is refused for it any more. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/compile/mod.rs | 91 +++++++++++++++++++++++---------------- tests/grain/corpus/mod.rs | 29 +++++++++++++ tests/grain/format.rs | 21 +++++++-- 3 files changed, 102 insertions(+), 39 deletions(-) diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index ecdd2a9bf..40c93845a 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -7,7 +7,7 @@ use core::mem; use std::prelude::v1::*; use crate::ast::{ - ASTFlags, ASTNode, Expr, FlowControl, FnCallExpr, OpAssignment, ScriptFuncDef, Stmt, StmtBlock, + ASTFlags, Expr, FlowControl, FnCallExpr, OpAssignment, ScriptFuncDef, Stmt, StmtBlock, SwitchCasesCollection, }; use crate::tokenizer::Token; @@ -347,9 +347,7 @@ impl Lowering { // `this` is deliberately not in the second class. Rhai reaches it // through the caller's `&mut` (`eval/chaining.rs:528`), so a method // step that mutates lands in the caller's value — walking a copy would - // drop the write silently, which is the one failure mode worth - // refusing outright. It is also unreachable: a body that mentions - // `this` is never compiled to a chunk at all. + // drop the write silently. It gets a root of its own instead. let root_spec = match root { Expr::Variable(v, ..) if !has_namespace!(v) => match self.slots.resolve(&v.1) { Some(slot) => Root::Local { @@ -370,9 +368,10 @@ impl Lowering { }, None => return false, }, + Expr::ThisPtr(pos) => Root::This { pos: *pos }, // A qualified root resolves against imported modules, which need // `import` — the escape hatch's job. - Expr::Variable(..) | Expr::ThisPtr(..) => return false, + Expr::Variable(..) => return false, _ if matches!(tail, Tail::Read) => Root::Temporary, // Unreachable through the parser, which refuses `f().x = 1` // outright (`eval/chaining.rs:559`). @@ -670,14 +669,6 @@ impl Lowering { /// The body runs in a fresh scope with the parameters already pushed /// (`func/script.rs:73`), so the parameters are exactly slots 0 upwards. fn function(&mut self, def: &ScriptFuncDef) -> Option { - // `this` is not a scope entry, so no slot addresses it. The declared - // type is not enough to go on: an untyped `fn double() { this * 2 }` - // leaves `this_type` empty and still needs one, so the body is what - // has to be checked. - if def.this_type.is_some() || uses_this(def.body.statements()) { - return None; - } - let first_op = self.code.len(); let saved_slots = mem::take(&mut self.slots); let saved_loops = mem::take(&mut self.loops); @@ -783,6 +774,27 @@ impl Lowering { // lowering rather than becoming a fragment. Stmt::FnCall(call, ..) if call.name == crate::engine::KEYWORD_EVAL => false, + // `this` on the left. Ahead of the two variable arms because rhai's + // parser puts it there too (`parser.rs:2002`), and because the + // chain arm below would otherwise take `this.x = 1`'s sibling. + Stmt::Assignment(payload) if matches!(&payload.1.lhs, Expr::ThisPtr(..)) => { + let (op_info, binary) = &**payload; + + // Before the right-hand side, not after. Rhai checks that + // `this` is bound and returns before it evaluates the value + // (`eval/stmt.rs:300-303`) — unlike the variable arm, which + // evaluates first — so an unbound `this = nosuch` is + // `ErrorUnboundThis` and not the value's own failure. + self.emit_at(Op::RequireThis, binary.lhs.position()); + + self.expression(&binary.rhs); + let op = self.op_assignment(op_info); + + self.emit_at(Op::AssignThis { op }, op_info.position()); + self.emit(Op::Unit); + true + } + // A plain local on the left. Stmt::Assignment(payload) if matches!(&payload.1.lhs, Expr::Variable(v, ..) @@ -1462,10 +1474,14 @@ impl Lowering { // guarded arms above fall through to here when their guard fails — // a pool-defeating constant, a literal too long for its operand, a // call rhai resolves syntactically. + // The frame's receiver, flattened as every consumer but three + // wants it — see [`Op::LoadThis`] and `unflattened` below. Its own + // position, because that is what `ErrorUnboundThis` carries. + Expr::ThisPtr(pos) => self.emit_at(Op::LoadThis, *pos), + Expr::Coalesce(..) | Expr::MethodCall(..) | Expr::Property(..) - | Expr::ThisPtr(..) | Expr::DynamicConstant(..) | Expr::FnCall(..) | Expr::Array(..) @@ -1483,6 +1499,17 @@ impl Lowering { return None; } + // `f(this, ..)` takes the same rewrite as a variable. Rhai also requires + // the receiver not to be shared and nothing to be curried + // (`func/call.rs:1417`), and neither is a question the compiler can + // answer: sharing is a run-time property, deferred to the VM as it + // already is for a read-only local, and a curried redirect can never + // reach this instruction because `call`/`curry` go through + // `Op::CallFnPtr` and `is_lowerable_call` refuses them here. + if let Some(Expr::ThisPtr(..)) = call.args.first() { + return Some(Receiver::This); + } + let Some(Expr::Variable(payload, ..)) = call.args.first() else { return None; }; @@ -1504,6 +1531,16 @@ impl Lowering { // `f(x, ..)` is `x.f(..)`, so the variable is read after the other // arguments and by reference. See [`Op::CallRef`]. if let Some(receiver) = self.receiver(call) { + // `this` goes on *first*, unlike either of the others. Rhai's two + // arms disagree about when it is read: the by-reference one takes a + // pointer after the arguments (`func/call.rs:1417`), but the + // fallback a shared or unbound receiver lands in reads and flattens + // it before them (`:1462`). Reading first is what makes an unbound + // `f(this, nosuch)` report `ErrorUnboundThis`, and what stops an + // argument that writes to `this` being seen by the value passed. + if let Receiver::This = receiver { + self.emit_at(Op::LoadThis, call.args[0].position()); + } for arg in call.args.iter().skip(1) { self.expression(arg); } @@ -1595,6 +1632,10 @@ impl Lowering { None => self.expression(expr), } } + // The receiver can be a shared cell too — a closure capturing the + // variable a method was called on — and the three readers that come + // through here have to see the cell rather than what it holds. + Expr::ThisPtr(pos) => self.emit_at(Op::LoadThisShared, *pos), other => self.expression(other), } } @@ -2052,28 +2093,6 @@ fn flatten_chain(expr: &Expr) -> Option<(&Expr, Vec>)> { Some((root, steps)) } -/// Whether a function body reaches for `this`. -/// -/// Such a body can still be *called* — rhai keeps its own copy and the call -/// goes through dispatch — it just cannot be a chunk, because a chunk's locals -/// are scope slots and `this` is not in the scope. -fn uses_this(statements: &[Stmt]) -> bool { - let mut found = false; - let path = &mut Vec::new(); - for stmt in statements { - // `walk` hands over the path to the node it is visiting, so the node - // itself is the last entry. - stmt.walk(path, &mut |path| { - if matches!(path.last(), Some(ASTNode::Expr(Expr::ThisPtr(..)))) { - found = true; - } - // Stop descending once the answer is settled. - !found - }); - } - found -} - /// Wrap statements as a block expression. /// /// `Expr::Stmt` is the one shape `eval_expression_tree_raw` routes to diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index 1922cae4e..c576e5137 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -523,4 +523,33 @@ pub const CASES: &[Case] = &[ case("host_mutation_before_a_failure_survives", "let w = widget(1); try { w.bump_then_fail(); } catch(e) {} w.level"), case("host_mutation_before_a_failure_survives_in_a_map", "let m = #{ w: widget(1) }; try { m.w.bump_then_fail(); } catch(e) {} m.w.level"), case("host_mutation_before_a_failure_survives_in_an_array", "let a = [widget(1)]; try { a[0].bump_then_fail(); } catch(e) {} a[0].level"), + // `this`, which is a register rather than a scope entry and so is reached + // by instructions of its own. + case("this_read", "fn get() { this } let v = 7; v.get()"), + case("this_in_an_expression", "fn double() { this * 2 } let v = 21; v.double()"), + case("this_assign", "fn set() { this = 9; } let v = 1; v.set(); v"), + case("this_op_assign", "fn bump(n) { this += n; } let v = 1; v.bump(4); v"), + case("this_op_assign_on_a_string", "fn add(s) { this += s; } let v = \"a\"; v.add(\"b\"); v"), + case("this_is_the_bodys_value", "fn twice() { this + this } let v = 4; v.twice()"), + // Never inherited: a plain call from a bound body gets no receiver. + case("error_this_is_not_inherited", "fn outer() { inner() } fn inner() { this } let v = 1; v.outer()"), + case("error_this_unbound_in_call_style", "fn get() { this } get()"), + // The check precedes the right-hand side, unlike the variable arm. + case("error_this_assign_unbound_beats_a_bad_value", "fn set() { this = nosuch; } set()"), + // Chains rooted at `this`, which must write back into the caller's value. + case("this_property", "fn count() { this.n } let m = #{ n: 5 }; m.count()"), + case("this_property_assign", "fn set() { this.n = 9; } let m = #{ n: 1 }; m.set(); m.n"), + case("this_index", "fn first() { this[0] } let a = [3, 4]; a.first()"), + case("this_index_assign", "fn set() { this[0] = 9; } let a = [1, 2]; a.set(); a"), + case("this_method_step", "fn grow() { this.push(3); } let a = [1, 2]; a.grow(); a"), + case("this_host_method", "fn raise() { this.bump(); } let w = widget(4); w.raise(); w.level"), + case("this_host_property", "fn read() { this.level } let w = widget(4); w.read()"), + // A method on `this` that reaches another compiled function. + case("this_nested_method", "fn outer() { this.inner() } fn inner() { this * 2 } let v = 5; v.outer()"), + // `f(this, ..)`, which rhai rewrites to `this.f(..)` by reference. + case("this_as_first_argument", "fn grow() { push(this, 3); } let a = [1, 2]; a.grow(); a"), + case("this_as_first_argument_pure", "fn size() { len(this) } let a = [1, 2]; a.size()"), + case("this_as_a_later_argument", "fn plus(n) { n + this } let v = 1; v.plus(2)"), + // Arity excludes the receiver, so these are two different functions. + case("this_method_arity", "fn f() { 1 } fn f(x) { this + x } let v = 10; [v.f(), v.f(5)]"), ]; diff --git a/tests/grain/format.rs b/tests/grain/format.rs index 63a82367a..3a32edfc8 100644 --- a/tests/grain/format.rs +++ b/tests/grain/format.rs @@ -367,16 +367,31 @@ fn script_functions_survive_the_round_trip() { /// depends on rhai's copy cannot be written — silently dropping it would /// produce an artifact that loads and then cannot find its own function. #[test] +#[cfg(not(feature = "no_module"))] fn a_function_the_compiler_cannot_lower_refuses_to_write() { let engine = corpus::engine(); - // `this` is not a scope entry, so no slot addresses it. - let ast = engine.compile("fn double() { this * 2 } let x = 21; x.double()").expect("must compile"); + // `import` declares into the caller's scope, which the slot model cannot + // account for. `this` used to be the example here, and is not one any more. + let ast = engine.compile(r#"fn m() { import "x" as y; 1 } m()"#).expect("must compile"); let program = Compiler::new().compile(&ast); - assert!(program.functions().is_empty(), "a body using `this` must not become a chunk",); + assert!(program.functions().is_empty(), "a body the slot model cannot account for must not become a chunk",); assert!(matches!(program.write(), Err(WriteError::HasScriptFunctions | WriteError::HasResiduals { .. }),), "got {:?}", program.write(),); } +/// The counterpart, and the milestone: a body that uses `this` is a chunk now, +/// so a program full of them is an artifact rather than a tree. +#[test] +fn a_body_using_this_is_compiled_and_writable() { + let engine = corpus::engine(); + let ast = engine.compile("fn bump(n) { this += n; this } let x = 21; x.bump(21); x").expect("must compile"); + let program = Compiler::new().compile(&ast); + + assert_eq!(program.residual_count(), 0, "{:?}", program.first_unsupported()); + assert!(!program.functions().is_empty(), "a body using `this` must become a chunk"); + assert!(program.write().is_ok(), "got {:?}", program.write()); +} + /// A program with one of everything the corruption tests need to reach: a /// float constant, a loop, and a `switch` — whose table is the one part of an /// artifact holding jump targets that are not in the code, and so the one the From a9d83406eb36d89ef9d1438152917435e6c1a280 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:30:03 -0700 Subject: [PATCH 26/33] check a bound handler end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shape the work exists for: call_fn_with_options with bind_this_ptr on a handler that reads and writes this, compared against the same call through the walker on both the value and the caller's Dynamic afterwards. Including the write that precedes a throw, which rhai keeps. Two tests had gone hollow: their example of something the compiler cannot lower was a body using this, which is no longer one. They now use a bare eval statement, and assert that premise rather than assuming it — eval in expression position leaves a fragment instead and would have left the function compiled. Co-Authored-By: Claude Opus 5 (1M context) --- tests/grain/call_fn.rs | 103 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 11 deletions(-) diff --git a/tests/grain/call_fn.rs b/tests/grain/call_fn.rs index 6e7bf7af1..b0d835fe1 100644 --- a/tests/grain/call_fn.rs +++ b/tests/grain/call_fn.rs @@ -1,10 +1,17 @@ -//! What a program contributes to `global` while [`Vm::call_fn`] runs. +//! Calling one compiled function, with the environment and receiver rhai gives +//! it. //! //! A program's library, source and module resolver used to be installed around //! its *main chunk* only, so a function reached through `call_fn` ran without //! them. That is not a corner: the compiler leaves anything it cannot lower as //! an AST in the library, and rhai finds it only in `global.lib`. +//! +//! The other half is `this`. An event handler bound to its state through +//! `bind_this_ptr` is the common shape of a `call_fn` caller, and the whole of +//! what these check is that a `Vm` answers such a call the way an `Engine` +//! does — including that a write through `this` lands in the caller's value. +use rhai::grain::format::WriteError; use rhai::grain::{Compiler, Vm}; use rhai::{CallFnOptions, Dynamic, Engine, EvalAltResult, Scope, INT}; @@ -13,20 +20,26 @@ fn holder(count: INT) -> Dynamic { Dynamic::from_map([("count".into(), Dynamic::from(count))].into_iter().collect()) } -/// `this` is what keeps `inner` an AST in the program's library, which is the -/// only way to observe from outside whether the library is installed. -const PROGRAM: &str = " - fn labelled() { this.count } +/// A bare `eval` statement is what keeps the callee an AST in the program's +/// library — it can declare into the caller's scope, which no slot model can +/// account for — and the library is the only thing that can answer the method +/// call below. In *expression* position it becomes a fragment instead, which +/// leaves the function compiled and would not exercise this at all. +/// +/// It used to be `this` here. That stopped being unlowerable, which would have +/// left this test passing while checking nothing. +const PROGRAM: &str = r#" + fn labelled() { eval("1"); 42 } fn outer(m) { m.labelled() } -"; +"#; /// The same shape, but the un-lowered function's name is also a rhai built-in /// (`Dynamic::tag`). Without the library the call does not fail — it silently /// resolves to the built-in getter and answers 0. -const SHADOWED: &str = " - fn tag() { this.count } +const SHADOWED: &str = r#" + fn tag() { eval("1"); 42 } fn outer(m) { m.tag() } -"; +"#; fn walked(engine: &Engine, source: &str) -> INT { let ast = engine.compile(source).unwrap(); @@ -36,20 +49,24 @@ fn walked(engine: &Engine, source: &str) -> INT { fn run(engine: &Engine, source: &str) -> INT { let ast = engine.compile(source).unwrap(); let program = Compiler::new().compile(&ast); + // The premise: the callee really is one the compiler left behind, so the + // library has to be installed for the call to resolve at all. Refusing to + // write for *that* reason is the public way to see it. + assert!(matches!(program.write(), Err(WriteError::HasScriptFunctions)), "{source:?} no longer keeps a library, so this would prove nothing (write said {:?})", program.write(),); Vm::new(engine).call_fn(&mut Scope::new(), &program, "outer", (holder(7),)).unwrap() } #[test] fn a_call_reaches_a_function_the_compiler_left_to_rhai() { let engine = Engine::new(); - assert_eq!(walked(&engine, PROGRAM), 7); + assert_eq!(walked(&engine, PROGRAM), 42); assert_eq!(run(&engine, PROGRAM), walked(&engine, PROGRAM)); } #[test] fn a_missing_library_cannot_be_answered_by_a_builtin_of_the_same_name() { let engine = Engine::new(); - assert_eq!(walked(&engine, SHADOWED), 7); + assert_eq!(walked(&engine, SHADOWED), 42); assert_eq!(run(&engine, SHADOWED), walked(&engine, SHADOWED)); } @@ -103,3 +120,67 @@ fn an_error_inside_a_call_carries_the_program_source() { other => panic!("expected a wrapped call error, got {other:?}"), } } + +/// Both sides of one `call_fn` with a bound receiver: the value it answers, and +/// the caller's `Dynamic` afterwards. +fn bound(engine: &Engine, source: &str, name: &str, state: &mut Dynamic) -> Result { + let ast = engine.compile(source).unwrap(); + let program = Compiler::new().compile(&ast); + + Vm::new(engine) + .call_fn_with_options(CallFnOptions::new().bind_this_ptr(state), &mut Scope::new(), &program, name, (1 as INT,)) + .map_err(|err| format!("{err:?}")) +} + +/// The same through the walker, for the comparison to mean anything. +fn walked_bound(engine: &Engine, source: &str, name: &str, state: &mut Dynamic) -> Result { + let ast = engine.compile(source).unwrap(); + engine + .call_fn_with_options(CallFnOptions::new().bind_this_ptr(state), &mut Scope::new(), &ast, name, (1 as INT,)) + .map_err(|err| format!("{err:?}")) +} + +/// The shape this work exists for: a handler with its state on `this`. +#[test] +fn a_handler_bound_to_its_state_answers_as_the_walker_does() { + let engine = Engine::new(); + const HANDLER: &str = "fn bump(n) { this.count += n; this.count }"; + + let mut state = holder(1); + let mut walked_state = holder(1); + + assert_eq!(bound(&engine, HANDLER, "bump", &mut state), Ok(2)); + assert_eq!(walked_bound(&engine, HANDLER, "bump", &mut walked_state), Ok(2)); + // And the write reached the caller's own `Dynamic`, on both sides. + assert_eq!(format!("{state:?}"), format!("{walked_state:?}")); + assert_eq!(format!("{state:?}"), r#"#{"count": 2}"#); +} + +/// Rhai reaches `this` through the caller's storage, so a handler that writes +/// and then fails has already written. +#[test] +fn a_write_before_a_failure_reaches_the_caller() { + let engine = Engine::new(); + const HANDLER: &str = r#"fn bump(n) { this.count += n; throw "boom" }"#; + + let mut state = holder(1); + let mut walked_state = holder(1); + + assert!(bound(&engine, HANDLER, "bump", &mut state).is_err()); + assert!(walked_bound(&engine, HANDLER, "bump", &mut walked_state).is_err()); + + assert_eq!(format!("{state:?}"), format!("{walked_state:?}")); + assert_eq!(format!("{state:?}"), r#"#{"count": 2}"#); +} + +/// A handler is a chunk now, so a program full of them is an artifact. +#[test] +fn a_program_of_handlers_is_writable() { + let engine = Engine::new(); + let ast = engine.compile("fn on_open(n) { this.count += n } fn on_close() { this.count = 0 }").unwrap(); + let program = Compiler::new().compile(&ast); + + assert_eq!(program.residual_count(), 0); + assert_eq!(program.functions().len(), 2); + assert!(program.write().is_ok(), "got {:?}", program.write()); +} From 866568e9c3a58efe8017fccabdb4aa853372fb29 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:41:47 -0700 Subject: [PATCH 27/33] bind `obj.call(f)`'s receiver by reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rhai binds obj as the closure's this by reference (func/call.rs:862), so a closure that writes to this writes to obj. The operand stack only ever held a copy, so Op::CallFnPtr now carries where the receiver came from and the write is carried back there. A temporary has nowhere to go, which is what rhai does too. The receiver is pushed unflattened: one that is a shared cell arrives as the cell, so the write lands where every holder can see it and no write-back is needed — run_chain's rule for a chain root. This also repairs a regression from lowering `this` bodies. A closure using `this` became a chunk, invisible to rhai's dispatch, while call_fn_ptr refused the compiled path whenever a receiver was present — so `v.call(|| this * 2)` failed outright with ErrorFunctionNotFound. Nothing in the suite covered it; six corpus cases do now. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/bytecode/code.rs | 88 ++++++++++++++++++-- src/grain/bytecode/op.rs | 12 +++ src/grain/bytecode/verify.rs | 4 + src/grain/compile/mod.rs | 48 ++++++++++- src/grain/vm/mod.rs | 150 +++++++++++++++++++++++++++++------ tests/grain/corpus/mod.rs | 17 ++++ 6 files changed, 285 insertions(+), 34 deletions(-) diff --git a/src/grain/bytecode/code.rs b/src/grain/bytecode/code.rs index ca7ee2b66..d6ede071f 100644 --- a/src/grain/bytecode/code.rs +++ b/src/grain/bytecode/code.rs @@ -161,6 +161,12 @@ pub mod tag { pub const ASSIGN_THIS_OP: u8 = 0x3d; /// [`Op::CallRef`](super::Op::CallRef) through [`Receiver::This`](super::Receiver::This). pub const CALL_THIS_REF: u8 = 0x3e; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) on a local, which is written back to. + pub const CALL_FN_PTR_ON_LOCAL: u8 = 0x3f; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) on a variable no slot names. + pub const CALL_FN_PTR_ON_NAMED: u8 = 0x40; + /// [`Op::CallFnPtr`](super::Op::CallFnPtr) on the frame's receiver. + pub const CALL_FN_PTR_ON_THIS: u8 = 0x41; } /// How wide each tag's instruction is, with 0 for the tags that are not one. @@ -213,6 +219,12 @@ static WIDTHS: [u8; 256] = { widths[tag::ASSIGN_THIS as usize] = 1; widths[tag::ASSIGN_THIS_OP as usize] = 3; widths[tag::CALL_THIS_REF as usize] = 4; + + // The receiver's value is on the stack for all of these; only where it came + // from differs, and only two of them need an operand to say it. + widths[tag::CALL_FN_PTR_ON_LOCAL as usize] = 4; + widths[tag::CALL_FN_PTR_ON_NAMED as usize] = 4; + widths[tag::CALL_FN_PTR_ON_THIS as usize] = 2; widths[tag::MAKE_CLOSURE as usize] = 3; widths[tag::PUSH_HANDLER as usize] = 5; widths[tag::PUSH_HANDLER_VAR as usize] = 7; @@ -548,13 +560,29 @@ pub fn assemble(ops: &[Op]) -> Result<(Vec, Vec), AssembleError> { code.push(tag::CURRY); code.push(*argc); } - Op::CallFnPtr { argc, method } => { - code.push(if *method { - tag::CALL_FN_PTR_METHOD - } else { - tag::CALL_FN_PTR - }); + Op::CallFnPtr { + argc, + method, + receiver, + } => { + // The receiver's value is on the stack whichever of these it + // is; the tag says where it came from, and two of them carry + // enough to reach it again. + let operand = match receiver { + Some(Receiver::Local(slot)) => Some((tag::CALL_FN_PTR_ON_LOCAL, *slot)), + Some(Receiver::Named(var)) => { + Some((tag::CALL_FN_PTR_ON_NAMED, small(*var as usize, "names")?)) + } + Some(Receiver::This) => Some((tag::CALL_FN_PTR_ON_THIS, 0)), + None if *method => Some((tag::CALL_FN_PTR_METHOD, 0)), + None => Some((tag::CALL_FN_PTR, 0)), + }; + let (tag, operand) = operand.expect("every arm answers"); + code.push(tag); code.push(*argc); + if matches!(tag, tag::CALL_FN_PTR_ON_LOCAL | tag::CALL_FN_PTR_ON_NAMED) { + code.extend_from_slice(&operand.to_le_bytes()); + } } Op::InterpolateStart => code.push(tag::INTERPOLATE_START), @@ -686,7 +714,11 @@ fn encoded_width(op: &Op) -> usize { | Op::RequireThis | Op::AssignThis { op: None } | Op::Return => 1, - Op::Curry(..) | Op::CallFnPtr { .. } | Op::Rotate(..) => 2, + Op::Curry(..) | Op::Rotate(..) => 2, + Op::CallFnPtr { receiver, .. } => match receiver { + Some(Receiver::Local(..) | Receiver::Named(..)) => 4, + Some(Receiver::This) | None => 2, + }, Op::Const(..) | Op::LoadLocal(..) | Op::StoreLocal(..) @@ -851,10 +883,27 @@ pub fn decode(code: &[u8], at: usize) -> Option { tag::CALL_FN_PTR => Op::CallFnPtr { argc: code[at + 1], method: false, + receiver: None, }, tag::CALL_FN_PTR_METHOD => Op::CallFnPtr { argc: code[at + 1], method: true, + receiver: None, + }, + tag::CALL_FN_PTR_ON_LOCAL => Op::CallFnPtr { + argc: code[at + 1], + method: true, + receiver: Some(Receiver::Local(small(2)?)), + }, + tag::CALL_FN_PTR_ON_NAMED => Op::CallFnPtr { + argc: code[at + 1], + method: true, + receiver: Some(Receiver::Named(u32::from(small(2)?))), + }, + tag::CALL_FN_PTR_ON_THIS => Op::CallFnPtr { + argc: code[at + 1], + method: true, + receiver: Some(Receiver::This), }, tag::INTERPOLATE_START => Op::InterpolateStart, tag::INTERPOLATE_APPEND => Op::InterpolateAppend, @@ -979,6 +1028,31 @@ mod tests { argc: 2, receiver: Receiver::This, }, + Op::CallFnPtr { + argc: 1, + method: false, + receiver: None, + }, + Op::CallFnPtr { + argc: 1, + method: true, + receiver: None, + }, + Op::CallFnPtr { + argc: 1, + method: true, + receiver: Some(Receiver::Local(4)), + }, + Op::CallFnPtr { + argc: 1, + method: true, + receiver: Some(Receiver::Named(5)), + }, + Op::CallFnPtr { + argc: 1, + method: true, + receiver: Some(Receiver::This), + }, Op::LoadThis, Op::LoadThisShared, Op::RequireThis, diff --git a/src/grain/bytecode/op.rs b/src/grain/bytecode/op.rs index 4522cf39e..e7498d863 100644 --- a/src/grain/bytecode/op.rs +++ b/src/grain/bytecode/op.rs @@ -384,6 +384,18 @@ pub enum Op { argc: u8, /// Whether the call is in method position (`f.call(x)`). method: bool, + /// Where the receiver came from, when there is anywhere to put it back. + /// + /// `obj.call(f)` binds `obj` as the closure's `this` **by reference** + /// (`func/call.rs:862`), so a closure that writes to `this` writes to + /// `obj`. The receiver's *value* is on the operand stack either way — + /// this only says where it came from, so the write can be carried back + /// there. + /// + /// `None` in call position, and for a receiver with nowhere to write + /// back to: `[1, 2].call(f)` mutates a temporary, as it does in rhai. + /// Only meaningful when `method` is set. + receiver: Option, }, /// Push an empty buffer for an interpolated string to be built in. diff --git a/src/grain/bytecode/verify.rs b/src/grain/bytecode/verify.rs index 0b8b8a284..6adba7a32 100644 --- a/src/grain/bytecode/verify.rs +++ b/src/grain/bytecode/verify.rs @@ -533,6 +533,10 @@ fn check_indices(at: usize, code: &[u8], pools: Pools) -> Result<(), VerifyError // omitting this would hand `program.assign_op` an unchecked index out // of a corrupt artifact. tag::ASSIGN_THIS_OP => bounded(index(1), "op-assignment", pools.assign_ops), + // The receiver's name, which the write-back resolves the scope entry + // by. A local's slot is not a pool index and is checked against the + // scope when it runs, as every other slot is. + tag::CALL_FN_PTR_ON_NAMED => bounded(index(2), "name", pools.names), tag::EVAL_AST | tag::EVAL_AST_KEEP => bounded(index(1), "fragment", pools.residuals), tag::CHAIN => { bounded(index(1), "chain", pools.chains.len())?; diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index 40c93845a..f8b548c4f 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -1420,7 +1420,21 @@ impl Lowering { let Expr::MethodCall(method, ..) = &binary.rhs else { unreachable!("checked by the guard"); }; - self.expression(&binary.lhs); + // `obj.call(f)` binds `obj` as the closure's `this` by + // reference (`func/call.rs:862`), so a write inside the closure + // has to reach `obj`. The value goes on the stack as it always + // did; the receiver says where to carry a write back to. + // + // Unflattened, for the reason `unflattened` gives: a receiver + // that is a shared cell has to arrive *as* the cell, so a write + // lands where every holder can see it and no write-back is + // needed at all. + let receiver = self.fn_ptr_receiver(&binary.lhs); + if receiver.is_some() { + self.unflattened(&binary.lhs); + } else { + self.expression(&binary.lhs); + } for arg in method.args.iter() { self.expression(arg); } @@ -1438,7 +1452,14 @@ impl Lowering { // see `fn_ptr_call`. The two disagreeing is deliberate. let pos = binary.rhs.position(); if method.name == "call" { - self.emit_at(Op::CallFnPtr { argc, method: true }, pos); + self.emit_at( + Op::CallFnPtr { + argc, + method: true, + receiver, + }, + pos, + ); } else { self.emit_at(Op::Curry(argc), pos); } @@ -1489,6 +1510,27 @@ impl Lowering { } } + /// Where `obj.call(f)`'s receiver came from, when a write through the + /// closure's `this` has somewhere to land. + /// + /// `None` for anything rhai would evaluate into a temporary — `[1, 2].call(f)` + /// mutates a copy in the walker too, so there is nothing to carry back. + fn fn_ptr_receiver(&mut self, receiver: &Expr) -> Option { + match receiver { + Expr::ThisPtr(..) => Some(Receiver::This), + Expr::Variable(payload, ..) if !has_namespace!(payload) => { + match self.slots.resolve(&payload.1) { + Some(slot) => Some(Receiver::Local(slot)), + None if self.is_variable_name(&payload.1, false) => { + Some(Receiver::Named(self.push_name(payload.1.clone()))) + } + None => None, + } + } + _ => None, + } + } + /// Where rhai's method-call rewrite would take this call's first argument /// from, if it applies at all (`func/call.rs:1434`). fn receiver(&mut self, call: &FnCallExpr) -> Option { @@ -1694,6 +1736,8 @@ impl Lowering { Op::CallFnPtr { argc: (argc - 1) as u8, method: false, + // Call position binds no receiver at all. + receiver: None, }, pos, ); diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index f2f306ee3..fcbe9bbee 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -1711,11 +1711,15 @@ impl<'e> Vm<'e> { /// so the name is matched against it first and only the miss goes to /// `call_raw`. #[inline(never)] + #[allow(clippy::too_many_arguments)] fn call_fn_ptr( &mut self, program: &Program, argc: usize, method: bool, + receiver: Option, + scope: &mut Scope, + frame_base: usize, pos: Position, ) -> VmResult { let base = self @@ -1730,9 +1734,9 @@ impl<'e> Vm<'e> { // is how a closure is called against a `this`. Rhai reports the // mismatch against that argument, not against the target, which is why // the position moves with it. - let mut this = None; + let mut receiver_at = None; if method && !self.stack[at].is::() { - this = Some(self.stack[at].clone()); + receiver_at = Some(at); at += 1; if at >= self.stack.len() { return Err(self.mismatch::(self.stack[at - 1].type_name(), pos)); @@ -1746,39 +1750,122 @@ impl<'e> Vm<'e> { let taken = self.stack.len() - at - 1; let curried = pointer.curry().len(); - let function = (this.is_none()) - .then(|| program.function_named(pointer.fn_name(), curried + taken)) - .flatten() + // A receiver does not change which function a pointer names, only what + // the callee's `this` is: rhai keys its own script pointers on the + // declared parameter count alone (`types/fn_ptr.rs:422`), and binds the + // receiver alongside them. + let function = program + .function_named(pointer.fn_name(), curried + taken) .map(|f| (f.params.clone(), f.chunk)); - if let Some((params, chunk)) = function { + // Bound once, whichever path takes it. Curried values are spliced in + // above `at`, so the receiver's index is unaffected either way. + let (mut bound, write_back) = match receiver_at { + Some(index) => { + let (value, write_back) = bind_this(&mut self.stack[index]); + (Some(value), write_back) + } + None => (None, false), + }; + + let outcome = if let Some((params, chunk)) = function { // Curried arguments go in front of the call's own, which is what // currying means and where the callee's parameters expect them. let first = at + 1; self.stack .splice(first..first, pointer.curry().iter().cloned()); - let value = - self.call_compiled(program, pointer.fn_name(), ¶ms, chunk, first, pos)?; - self.stack.truncate(base); - return Ok(value); + + let (result, returned) = self.call_compiled_with_this( + program, + pointer.fn_name(), + ¶ms, + chunk, + first, + pos, + bound.take(), + ); + bound = returned; + result + } else { + // Anything else is rhai's: a native function, a name registered + // elsewhere, or a pointer it built itself. + let mut args: Vec = self.stack.drain(at + 1..).collect(); + let context = native_context(self.engine, pointer.fn_name(), None, &self.global, pos); + pointer + .call_raw(&context, bound.as_mut(), &mut args) + .map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos); + } + err + }) + }; + + // Before `?`, as everywhere else: rhai binds the receiver by reference, + // so a closure that writes and then raises has already written. + if let Some(index) = receiver_at { + unbind_this(&mut self.stack[index], bound, write_back); + if write_back { + let updated = self.stack[index].clone(); + self.return_receiver(program, receiver, updated, scope, frame_base)?; + } } - // Anything else is rhai's: a native function, a name registered - // elsewhere, or a pointer it built itself. - let mut args: Vec = self.stack.drain(at + 1..).collect(); - let context = native_context(self.engine, pointer.fn_name(), None, &self.global, pos); - let value = pointer - .call_raw(&context, this.as_mut(), &mut args) - .map_err(|mut err| { - if err.position().is_none() { - err.set_position(pos); - } - err - })?; + let value = outcome?; self.stack.truncate(base); Ok(value) } + /// Carry a write through `obj.call(f)`'s `this` back to `obj` itself. + /// + /// Rhai binds the receiver by reference (`func/call.rs:862`), so the write + /// lands in the variable. The operand stack only ever held a copy of it, + /// and this is what puts the copy back where it came from. + /// + /// Nothing to do for a shared receiver: it arrived *as* the cell, so the + /// write already landed where every holder can see it — `run_chain`'s rule + /// for a chain root, and for the same reason. + fn return_receiver( + &mut self, + program: &Program, + receiver: Option, + value: Dynamic, + scope: &mut Scope, + frame_base: usize, + ) -> Result<(), Box> { + if is_shared!(value) || value.is_read_only() { + return Ok(()); + } + + match receiver { + Some(Receiver::Local(slot)) => { + let index = frame_base + slot as usize; + if index >= scope.len() { + return Err(malformed(format!("local slot {slot} is out of scope"))); + } + *scope.get_mut_by_index(index) = value; + } + Some(Receiver::Named(var)) => { + let name = program + .name(var) + .ok_or_else(|| malformed(format!("no name {var}")))?; + // A resolver's answer or a module constant has no entry behind + // it, and rhai could not have written through one either. + if let Some(entry) = scope.get_mut(name) { + *entry = value; + } + } + Some(Receiver::This) => { + if let Some(entry) = self.this.as_mut() { + *entry = value; + } + } + // A temporary, which rhai mutates a copy of too. + None => {} + } + Ok(()) + } + /// Concatenate the segments of an interpolated string, reproducing /// `eval/expr.rs:280-304`. /// @@ -3202,10 +3289,23 @@ impl<'e> Vm<'e> { self.stack.push(pointer.into()); } - code::tag::CALL_FN_PTR | code::tag::CALL_FN_PTR_METHOD => { + code::tag::CALL_FN_PTR + | code::tag::CALL_FN_PTR_METHOD + | code::tag::CALL_FN_PTR_ON_LOCAL + | code::tag::CALL_FN_PTR_ON_NAMED + | code::tag::CALL_FN_PTR_ON_THIS => { let argc = code[pc + 1] as usize; - let method = tag == code::tag::CALL_FN_PTR_METHOD; - let value = self.call_fn_ptr(program, argc, method, pos())?; + let method = tag != code::tag::CALL_FN_PTR; + let receiver = match tag { + code::tag::CALL_FN_PTR_ON_LOCAL => Some(Receiver::Local(small(2)?)), + code::tag::CALL_FN_PTR_ON_NAMED => { + Some(Receiver::Named(u32::from(small(2)?))) + } + code::tag::CALL_FN_PTR_ON_THIS => Some(Receiver::This), + _ => None, + }; + let value = + self.call_fn_ptr(program, argc, method, receiver, scope, base, pos())?; self.stack.push(value); } diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index c576e5137..5751fb971 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -552,4 +552,21 @@ pub const CASES: &[Case] = &[ case("this_as_a_later_argument", "fn plus(n) { n + this } let v = 1; v.plus(2)"), // Arity excludes the receiver, so these are two different functions. case("this_method_arity", "fn f() { 1 } fn f(x) { this + x } let v = 10; [v.f(), v.f(5)]"), + // `obj.call(f)` binds `obj` as the closure's `this` by reference, so a + // write inside the closure reaches `obj`. The operand stack only ever holds + // a copy of it, which is why the instruction carries where it came from. + // + // The pointer is scoped to a block throughout, as the other closure cases + // are: a compiled closure's `FnPtr` carries a name where rhai's carries the + // body and its environment, so one left in the scope compares unequal for a + // reason that has nothing to do with the call. + case("closure_call_on_a_local_writes_back", "let v = 21; { let f = || { this *= 2; }; v.call(f); } v"), + case("closure_call_on_a_local_inline", "let v = 21; v.call(|| { this *= 2; }); v"), + case("closure_call_on_a_local_reads", "let v = 21; let r = 0; { let f = || this * 2; r = v.call(f); } r"), + case("closure_call_mutates_an_array", "let a = [1]; { let f = || { this.push(2); }; a.call(f); } a"), + // And the receiver can be the frame's own receiver. + case("closure_call_on_this", "fn twice() { let f = || { this *= 2; }; this.call(f); } let v = 21; v.twice(); v"), + // A temporary receiver has nowhere to write back to, and rhai mutates a + // copy of it too. + case("closure_call_on_a_temporary", "let r = 0; { let f = || { this *= 2; }; r = (20 + 1).call(f); } r"), ]; From 643949d7447e1258feafde7e05fa896c645c4ff7 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:47:35 -0700 Subject: [PATCH 28/33] leave a `this`-taking chunk reachable by rhai MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A native calling a pointer back binds the element as the closure's this and appends what it likes beside it — map an index, reduce the running result — so the arity such a call arrives with is the native's business, not the callee's. A registered wrapper has exactly one arity and cannot cover that; rhai's own pointer carries the body and sizes the call from it. So wrappers skip these, and the library is kept when a program both makes pointers and has one. Repairs the second regression from lowering this bodies: [1,2,3].map(|| this * 2) failed with ErrorFunctionNotFound(anon (i64, i64)) — rhai asking for two arguments where the wrapper offered none. Four corpus cases. takes_this is derived from the chunk rather than encoded, as makes_fn_pointers is, so a loaded program and a compiled one cannot disagree about it. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/compile/mod.rs | 17 +++++++++++- src/grain/format/read.rs | 3 ++ src/grain/program.rs | 58 +++++++++++++++++++++++++++++++++++++-- src/grain/vm/callback.rs | 11 ++++++++ src/grain/vm/mod.rs | 1 + tests/grain/corpus/mod.rs | 9 ++++++ 6 files changed, 96 insertions(+), 3 deletions(-) diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index f8b548c4f..0bc81504b 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -155,6 +155,9 @@ impl Compiler { name: f.name, params: f.params, this_type: f.this_type, + // Derived from the chunk by `Program::new`, which is the one + // place that can see the assembled bytes. + takes_this: false, chunk: Chunk::new( offsets[f.first_op], offsets[f.first_op + f.op_count], @@ -167,7 +170,19 @@ impl Compiler { // for them: a function this compiler skipped, or a fragment that could // call one. With neither, every call resolves in the table above and // the library — an `AST`'s whole function tree — can be dropped. - let needs_walker = skipped > 0 || !lowering.residuals.is_empty(); + // + // The third case is a pointer to a `this`-taking chunk. Rhai reaches a + // compiled function through a registered wrapper, and a wrapper is + // registered at one arity — but a native calling a pointer against a + // receiver decides for itself how many arguments to append beside it, + // so no single arity is right. Rhai's own pointer carries the body and + // sizes the call from it, which is what its copy is kept here for. See + // `callback::wrappers`, which skips exactly these. + let escapes_as_pointer = crate::grain::program::makes_fn_pointers(&code) + && functions + .iter() + .any(|f| crate::grain::program::takes_this(&code, f.chunk, &lowering.chains)); + let needs_walker = skipped > 0 || !lowering.residuals.is_empty() || escapes_as_pointer; let mut program = Program::new( code.into(), diff --git a/src/grain/format/read.rs b/src/grain/format/read.rs index 1b8862ea5..68044c233 100644 --- a/src/grain/format/read.rs +++ b/src/grain/format/read.rs @@ -198,6 +198,9 @@ pub(super) fn read(bytes: &[u8]) -> Result, ReadError> { name, this_type, params, + // Not encoded: derived from the chunk by `Program::new`, so a loaded + // program and a compiled one cannot disagree about it. + takes_this: false, chunk: get_chunk(&mut cursor)?, }); } diff --git a/src/grain/program.rs b/src/grain/program.rs index f4510c3a8..be26ac161 100644 --- a/src/grain/program.rs +++ b/src/grain/program.rs @@ -7,7 +7,7 @@ use crate::module_resolvers::StaticModuleResolver; use crate::{ast::Expr, ast::Stmt, tokenizer::Token, Dynamic, ImmutableString, Module, Shared}; use crate::grain::bytecode::{ - AssignOp, Chain, Chunk, Code, Pools, Positions, Strings, Switch, TableError, + AssignOp, Chain, Chunk, Code, Op, Pools, Positions, Root, Strings, Switch, TableError, }; /// rhai's own `SharedModule`, which it does not re-export. @@ -45,6 +45,12 @@ pub struct Function { /// /// `None` for an ordinary function, which is nearly all of them. pub this_type: Option, + /// Whether the body reads or writes the frame's receiver. + /// + /// Derived from the chunk rather than encoded — see [`takes_this`]. It is + /// what keeps such a function reachable by rhai, which needs the body to + /// size a call a native makes through a pointer. + pub takes_this: bool, pub chunk: Chunk, } @@ -220,13 +226,52 @@ fn node_position(node: &ASTNode) -> rhai::Position { /// directly. Narrowing it would mean deciding where a pointer *goes*, which is /// a dataflow question over values that outlive the instruction that made /// them. Being wrong the other way loses a call at run time. -fn makes_fn_pointers(code: &[u8]) -> bool { +pub(crate) fn makes_fn_pointers(code: &[u8]) -> bool { use crate::grain::bytecode::code::tag; crate::grain::bytecode::disassemble(code) .any(|(at, ..)| matches!(code[at], tag::MAKE_CLOSURE | tag::MAKE_FN_PTR | tag::CURRY)) } +/// Whether a chunk reads or writes the frame's receiver. +/// +/// Read off the bytes for [`makes_fn_pointers`]'s reason: a program loaded from +/// an artifact never saw the body it came from. +/// +/// It decides whether rhai has to be able to reach the function itself. A +/// pointer to a `this`-taking chunk cannot go through a native wrapper — the +/// wrapper is registered at one arity, and how many arguments rhai will ask for +/// depends on what the *native* appends, which the wrapper cannot know. So such +/// a function is left to rhai, which has the body and can size the call itself. +pub(crate) fn takes_this(code: &[u8], chunk: Chunk, chains: &[Chain]) -> bool { + use crate::grain::bytecode::code::tag; + + let (entry, end) = (chunk.entry() as usize, chunk.end() as usize); + if end > code.len() || entry > end { + return false; + } + + crate::grain::bytecode::disassemble(&code[entry..end]).any(|(at, op)| { + match code[entry + at] { + tag::LOAD_THIS + | tag::LOAD_THIS_SHARED + | tag::REQUIRE_THIS + | tag::ASSIGN_THIS + | tag::ASSIGN_THIS_OP + | tag::CALL_THIS_REF + | tag::CALL_FN_PTR_ON_THIS => true, + // A chain says where it is rooted in the pool, not in the code. + tag::CHAIN => match op { + Op::Chain(index) => chains + .get(index as usize) + .map_or(false, |chain| matches!(chain.root, Root::This { .. })), + _ => false, + }, + _ => false, + } + }) +} + /// Everything a program holds besides its code, gathered so the constructor /// does not take ten positional arguments. pub(crate) struct Parts<'a> { @@ -253,6 +298,14 @@ impl<'a> Program<'a> { ) -> Self { let makes_fn_pointers = makes_fn_pointers(&code); let has_typed_methods = functions.iter().any(|f| f.this_type.is_some()); + + // Derived here so a program means the same whether it was compiled or + // loaded, and so the flag cannot disagree with the bytes it describes. + let mut functions = functions; + for function in &mut functions { + function.takes_this = takes_this(&code, function.chunk, &parts.chains); + } + let mut program = Self { code, main, @@ -671,6 +724,7 @@ mod tests { name, params: vec![0; argc], this_type, + takes_this: false, chunk: whole, }) .collect(); diff --git a/src/grain/vm/callback.rs b/src/grain/vm/callback.rs index b02310dd2..4eea529f2 100644 --- a/src/grain/vm/callback.rs +++ b/src/grain/vm/callback.rs @@ -75,6 +75,17 @@ pub(super) fn wrappers(program: &SharedProgram) -> Module { if arity > MAX_PARAMS { continue; } + // A `this`-taking chunk cannot be reached this way. A wrapper is + // registered at one arity, and how many arguments rhai asks for depends + // on what the *native* appends beside the receiver — `map` adds an + // index, `reduce` adds the running result — which the wrapper has no way + // to know. Rhai's own pointer carries the body and sizes the call from + // its declared arity (`types/fn_ptr.rs:501-535`); a name-only pointer, + // which is all a wrapper can be, cannot. So these are left to rhai, + // and `Program::needs_walker` is what keeps its copy alive for them. + if function.takes_this { + continue; + } let Some(name) = program.name(function.name) else { continue; }; diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index fcbe9bbee..463181a75 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -3531,6 +3531,7 @@ mod tests { name: index as u32, params: Vec::new(), this_type: None, + takes_this: false, chunk: Chunk::new(end_of(span.start), end_of(span.end), 8), }) .collect(); diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index 5751fb971..b6d6c819d 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -569,4 +569,13 @@ pub const CASES: &[Case] = &[ // A temporary receiver has nowhere to write back to, and rhai mutates a // copy of it too. case("closure_call_on_a_temporary", "let r = 0; { let f = || { this *= 2; }; r = (20 + 1).call(f); } r"), + // A native calling a pointer back against a receiver. How many arguments it + // appends beside the receiver is the native's business — `map` adds an + // index, `reduce` the running result — so no single wrapper arity is right + // and these have to stay reachable by rhai itself. + case("closure_map_binds_this", "[1, 2, 3].map(|| this * 2)"), + case("closure_filter_binds_this", "[1, 2, 3].filter(|| this > 1)"), + case("closure_for_each_binds_this", "let t = 0; [1, 2, 3].for_each(|| t += this); t"), + // And the argument form, which takes the element as a parameter instead. + case("closure_map_takes_an_argument", "[1, 2, 3].map(|x| x * 2)"), ]; From cd1855d4b92173daa5a3c01e884956a5f0ebe00d Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 13:59:05 -0700 Subject: [PATCH 29/33] compile and pass under unchecked and no_position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were already broken at the branch point — 10 errors and 2 — so neither is new, but neither built. no_position: reach for crate::types::Span rather than crate::types::position::Span, which the feature swaps out. unchecked: guard the limit checks the VM makes, since api/limits.rs is cfg'd out whole. On the test side both features change what a script *means*, so the affected cases are skipped rather than papered over: no_position has no table to strip and no two positions to tell apart, and unchecked turns 1/0 into a panic inside rhai's own built-in rather than an error. The corpus filter grew an unchecked arm, and format.rs's writable() now applies that filter — it was iterating every case regardless of build. Co-Authored-By: Claude Opus 5 (1M context) --- src/grain/compile/mod.rs | 5 +- src/grain/vm/mod.rs | 98 ++++++++++++++++++++++++------------- tests/grain/callback.rs | 2 + tests/grain/corpus/mod.rs | 7 +++ tests/grain/differential.rs | 3 ++ tests/grain/format.rs | 28 +++++++++-- tests/mod.rs | 5 ++ 7 files changed, 108 insertions(+), 40 deletions(-) diff --git a/src/grain/compile/mod.rs b/src/grain/compile/mod.rs index 0bc81504b..7a1707a04 100644 --- a/src/grain/compile/mod.rs +++ b/src/grain/compile/mod.rs @@ -2160,7 +2160,10 @@ fn flatten_chain(expr: &Expr) -> Option<(&Expr, Vec>)> { fn wrap_statements(statements: Vec) -> Expr { let span = statements.first().zip(statements.last()).map_or_else( || Span::new(Position::NONE, Position::NONE), - |(first, last)| crate::types::position::Span::new(first.position(), last.position()), + // `crate::types`, not `crate::types::position`: `no_position` swaps the + // module out for a zero-sized one and re-exports `Span` from whichever + // is in play. + |(first, last)| crate::types::Span::new(first.position(), last.position()), ); Expr::Stmt(Box::new(StmtBlock::new_with_span(statements, span))) diff --git a/src/grain/vm/mod.rs b/src/grain/vm/mod.rs index 463181a75..67c402667 100644 --- a/src/grain/vm/mod.rs +++ b/src/grain/vm/mod.rs @@ -3,6 +3,7 @@ use core::mem; use std::prelude::v1::*; use crate::engine::{FN_IDX_GET, FN_IDX_SET}; +#[cfg(not(feature = "unchecked"))] use crate::eval::calc_data_sizes; use crate::func::{get_builtin_binary_op_fn, get_builtin_op_assignment_fn}; use crate::packages::string_basic::print_with_func; @@ -1916,12 +1917,22 @@ impl<'e> Vm<'e> { // After every segment, against the running total — a script must not // be able to build a string past `max_string_size` and hand it over // whole. - self.engine.throw_on_size((0, 0, len)).map_err(|mut err| { - if err.position().is_none() { - err.set_position(pos); - } - err - }) + #[cfg(not(feature = "unchecked"))] + { + self.engine.throw_on_size((0, 0, len)).map_err(|mut err| { + if err.position().is_none() { + err.set_position(pos); + } + err + }) + } + // `unchecked` removes the limits, and with them the only reason to have + // measured. + #[cfg(feature = "unchecked")] + { + let _ = (len, pos); + Ok(()) + } } /// Start iterating a value, the way rhai's `for` does @@ -2277,11 +2288,14 @@ impl<'e> Vm<'e> { first: usize, pos: Position, ) -> VmResult { - if self.global.level > self.engine.max_call_levels() { - return Err(Box::new(EvalAltResult::ErrorStackOverflow(pos))); - } - if params.len() > self.engine.max_variables() { - return Err(Box::new(EvalAltResult::ErrorTooManyVariables(pos))); + #[cfg(not(feature = "unchecked"))] + { + if self.global.level > self.engine.max_call_levels() { + return Err(Box::new(EvalAltResult::ErrorStackOverflow(pos))); + } + if params.len() > self.engine.max_variables() { + return Err(Box::new(EvalAltResult::ErrorTooManyVariables(pos))); + } } // A fresh scope: a function sees its parameters and nothing else. @@ -2445,35 +2459,48 @@ impl<'e> Vm<'e> { self.sizes.push((0, 0, 0)); } - // Rhai skips the whole measurement when no limit could reject it, and - // measuring is a walk of the value — so this is the difference between - // free and proportional to what the literal holds. - if self.engine.max_string_size() == 0 - && self.engine.max_array_size() == 0 - && self.engine.max_map_size() == 0 + // `unchecked` removes every limit this could reject against, so the + // running total is never read and measuring it is pure cost. The push + // above still happens: the stack is frame-floored either way, and the + // instruction that drops it does not know which build it is in. + #[cfg(feature = "unchecked")] { - return Ok(()); + let _ = (map, pos); + Ok(()) } - let value = self - .stack - .last() - .ok_or_else(|| malformed("size check with no element".to_string()))?; - let delta = calc_data_sizes(value, true); + #[cfg(not(feature = "unchecked"))] + { + // Rhai skips the whole measurement when no limit could reject it, + // and measuring is a walk of the value — so this is the difference + // between free and proportional to what the literal holds. + if self.engine.max_string_size() == 0 + && self.engine.max_array_size() == 0 + && self.engine.max_map_size() == 0 + { + return Ok(()); + } - let total = self - .sizes - .last_mut() - .ok_or_else(|| malformed("size check outside a literal".to_string()))?; - *total = ( - total.0 + delta.0 + usize::from(!map), - total.1 + delta.1 + usize::from(map), - total.2 + delta.2, - ); + let value = self + .stack + .last() + .ok_or_else(|| malformed("size check with no element".to_string()))?; + let delta = calc_data_sizes(value, true); + + let total = self + .sizes + .last_mut() + .ok_or_else(|| malformed("size check outside a literal".to_string()))?; + *total = ( + total.0 + delta.0 + usize::from(!map), + total.1 + delta.1 + usize::from(map), + total.2 + delta.2, + ); - self.engine - .throw_on_size(*total) - .map_err(|err| positioned(err, pos)) + self.engine + .throw_on_size(*total) + .map_err(|err| positioned(err, pos)) + } } /// Drop what an escaping error skipped the unwind for. @@ -2567,6 +2594,7 @@ impl<'e> Vm<'e> { let name = program .name(index) .ok_or_else(|| malformed(format!("no name {index}")))?; + #[cfg(not(feature = "unchecked"))] if scope.len() >= self.engine.max_variables() { return Err(Box::new(EvalAltResult::ErrorTooManyVariables( program.position(target), diff --git a/tests/grain/callback.rs b/tests/grain/callback.rs index c53a2e689..2746bfc29 100644 --- a/tests/grain/callback.rs +++ b/tests/grain/callback.rs @@ -117,6 +117,7 @@ fn stock_rhai_does_the_same_to_its_own_native_pointers() { } #[test] +#[cfg(not(feature = "unchecked"))] fn a_callback_reaching_a_second_one_still_resolves() { // The limit is raised because reaching a chunk through rhai's dispatch // costs more call levels than reaching a script function does, and the @@ -150,6 +151,7 @@ fn an_error_inside_a_callback_still_arrives() { /// are stricter, never laxer. A program that would have run out of budget on /// the walker must not somehow keep going here. #[test] +#[cfg(not(feature = "unchecked"))] fn a_callback_costs_more_call_levels() { let deepest = |depth: usize, run: &dyn Fn(&Engine, &str) -> Result| { // `depth` closures nested through `map`, each one a boundary, over an diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index b6d6c819d..cbc5aa985 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -113,6 +113,13 @@ pub fn applies_to_this_build(name: &str) -> bool { if name.starts_with("import_") || name.starts_with("export_") { return false; } + // `unchecked` removes the arithmetic guards, so `1 / 0` panics inside + // rhai's own built-in rather than raising — there is no behaviour left for + // the two sides to agree on, and the case would take the process with it. + #[cfg(feature = "unchecked")] + if matches!(name, "error_divide_by_zero" | "error_temp_root_index_runs_first") { + return false; + } // No shared prefix to key on: a float literal is incidental to most of // these, which are about interpolation, ranges and operator errors. #[cfg(feature = "no_float")] diff --git a/tests/grain/differential.rs b/tests/grain/differential.rs index b203952e5..23728723c 100644 --- a/tests/grain/differential.rs +++ b/tests/grain/differential.rs @@ -96,6 +96,9 @@ fn harness_detects_a_real_difference() { assert_ne!(run_stock(&engine, "1 + 1"), run_stock(&engine, "1 + 2"), "differing results must compare unequal",); assert_ne!(run_stock(&engine, "1"), run_stock(&engine, "1.0"), "int and float must not compare equal",); assert_ne!(run_stock(&engine, "let a = 1; a"), run_stock(&engine, "1"), "differing leftover scope must compare unequal",); + // `no_position` compiles positions out, so there is no such thing as the + // same error at a different one and nothing here to detect. + #[cfg(not(feature = "no_position"))] assert_ne!(run_stock(&engine, "let a = [1]; a[9]"), run_stock(&engine, "let a = [1]; a[9]"), "the same error at a different position must compare unequal",); } diff --git a/tests/grain/format.rs b/tests/grain/format.rs index 3a32edfc8..3036a9b10 100644 --- a/tests/grain/format.rs +++ b/tests/grain/format.rs @@ -56,6 +56,7 @@ fn run_stock(engine: &Engine, source: &str) -> Outcome { fn writable(engine: &Engine) -> Vec<(&'static str, &'static str, Vec)> { corpus::CASES .iter() + .filter(|case| corpus::applies_to_this_build(case.name)) .filter_map(|case| { let ast = engine.compile(case.source).ok()?; let bytes = Compiler::new().compile(&ast).write().ok()?; @@ -109,10 +110,13 @@ fn the_round_trip_covers_something_worth_covering() { // removes from the language, so there is no branch left to cover. #[cfg(not(feature = "no_float"))] "float_arithmetic", - "shadowing_nested", // DeclareLocal and UnwindTo - "while_loop", // jumps, Tick, AssignLocal with an op - "loop_break_value", // backpatched jumps - "error_divide_by_zero", // a position that has to survive + "shadowing_nested", // DeclareLocal and UnwindTo + "while_loop", // jumps, Tick, AssignLocal with an op + "loop_break_value", // backpatched jumps + // A position that has to survive. `unchecked` turns the failure it + // rests on into a panic in rhai, so the case is not run at all there. + #[cfg(not(feature = "unchecked"))] + "error_divide_by_zero", "switch_range", // a switch table, and the hasher probe with it "switch_guard", // and one whose arms are a chain rather than a target "string_slice_read", // a range constant, which is a host type in `Dynamic` @@ -334,7 +338,11 @@ fn refusing_to_write_names_the_construct_responsible() { panic!("{source:?} must refuse to write"); }; assert_eq!(construct, expected, "for {source:?}"); + // Under `no_position` there is no "where" to say, and the naming half + // above is the part that still means something. + #[cfg(not(feature = "no_position"))] assert!(!pos.is_none(), "the refusal must say where: {err}"); + let _ = pos; assert!(err.to_string().contains(expected), "{err}"); } } @@ -350,6 +358,9 @@ fn script_functions_survive_the_round_trip() { "fn fib(n) { if n < 2 { n } else { fib(n - 1) + fib(n - 2) } } fib(6)", "fn first() { 1 } fn second(x) { first() + x } second(4)", // Failing inside a function has to keep rhai's wrapping and position. + // `unchecked` turns this into a panic in rhai's own built-in rather + // than an error, so there is nothing left here to compare. + #[cfg(not(feature = "unchecked"))] "fn bad(x) { x / 0 } bad(1)", ] { let ast = engine.compile(source).expect("must compile"); @@ -524,6 +535,9 @@ fn trailing_bytes_are_refused() { /// reason the patch exposes `track_operation`. A host running untrusted /// bytecode must set `max_operations`, exactly as it must for untrusted source. #[test] +// A corrupted chunk can loop, and `max_operations` is what stops it. Without +// limits this hangs rather than fails, which is worse than not running. +#[cfg(not(feature = "unchecked"))] fn no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk() { let writer = Engine::new(); let bytes = sample(&writer); @@ -558,7 +572,11 @@ fn no_single_bit_flip_can_panic_or_smuggle_a_bad_chunk() { /// The device is sent a stripped artifact and knows nothing about the source. /// It fails, and all it can say is which instruction. The host kept the table, /// and turns that back into the position rhai itself would have reported. +/// There is no table to strip under `no_position`, so nothing to resolve; and +/// the failure it turns on is a division by zero, which `unchecked` makes a +/// panic in rhai rather than an error. #[test] +#[cfg(not(any(feature = "no_position", feature = "unchecked")))] fn a_stripped_program_reports_an_address_the_host_can_resolve() { let engine = corpus::engine(); let source = "let a = 1;\nlet b = 0;\na / b"; @@ -595,6 +613,7 @@ fn a_stripped_program_reports_an_address_the_host_can_resolve() { /// Attaching another program's table would misreport every error rather than /// reporting none, which is strictly worse than having no table. #[test] +#[cfg(not(feature = "no_position"))] fn a_table_from_a_different_program_is_refused() { let engine = corpus::engine(); @@ -611,6 +630,7 @@ fn a_table_from_a_different_program_is_refused() { /// A stripped artifact that arrives with a table still in it is a contradiction /// the reader should not paper over. #[test] +#[cfg(not(feature = "no_position"))] fn an_artifact_carrying_a_mismatched_table_does_not_load() { let engine = corpus::engine(); let bytes = sample(&engine); diff --git a/tests/mod.rs b/tests/mod.rs index 1a1653e61..90168b007 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -10,7 +10,12 @@ mod grain { mod callback; mod differential; mod format; + // Both are about execution staying inside a bound, which `unchecked` + // removes outright — and the artifact fuzzer needs `max_operations` to stop + // a corrupted chunk looping forever rather than failing. + #[cfg(not(feature = "unchecked"))] mod fuzz; + #[cfg(not(feature = "unchecked"))] mod limits; // Prices rhai's own AST nodes, which are exported under `internals` only. #[cfg(feature = "internals")] From a88ba84314c0d327422762ca811f5eb7daec3c49 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 16:00:07 -0700 Subject: [PATCH 30/33] resolve a bare function name to a pointer regardless of always_search_scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bare script-function name is a function pointer, not a variable read, and the check for one sat behind always_search_scope in search_scope_only. That flag means "do not trust the parse-time variable indices" — a name resolving to a function has no index to distrust — so gating the check on it made the name stop resolving for the rest of the run as soon as anything set the flag. Stock rhai has this too, with nothing else involved: fn dbl(x) { x * 2 } eval("let m = 2;"); let f = dbl; f.call(4) reported dbl as an unknown variable while the same script without the eval returned 8. grain merely hits it constantly, because it sets the flag for any program still holding an AST fragment — and a bare function name is one. The check still runs only for a variable with no cached index, so a variable of the same name keeps winning over the function. Verified against rhai's own suite: 368 passed. Co-Authored-By: Claude Opus 5 (1M context) --- src/eval/expr.rs | 80 ++++++++++++++++++++++++----------------- tests/fn_ptr.rs | 23 ++++++++++++ tests/grain/callback.rs | 15 ++++++++ 3 files changed, 86 insertions(+), 32 deletions(-) diff --git a/src/eval/expr.rs b/src/eval/expr.rs index 06227fecb..168dc84a4 100644 --- a/src/eval/expr.rs +++ b/src/eval/expr.rs @@ -55,6 +55,54 @@ impl Engine { ) -> RhaiResultOf> { // Make sure that the pointer indirection is taken only when absolutely necessary. + // A bare script-function name is a function pointer, not a variable read. + // + // Checked ahead of `always_search_scope` rather than inside the match: + // that flag means "do not trust the parse-time variable indices", and a + // name resolving to a function has no index to distrust. Gating this on + // it made the name stop resolving for the rest of a run as soon as + // anything set it — an `eval` that changes the scope, a variable + // resolver that does, or a program still holding an AST fragment — so + // `fn f(x) { x } eval("let m = 1;"); [1].map(f)` reported `f` as an + // unknown variable while the same script without the `eval` worked. + // + // Still only for a variable with no cached index, which is what it was + // before: an index means the parser resolved the name to a real + // variable, and that variable keeps winning over a function sharing its + // name. + #[cfg(not(feature = "no_function"))] + if let Expr::Variable(v, None, ..) = expr { + if let Some(func) = global + .lib + .iter() + .flat_map(|m| m.iter_fn()) + .filter(|(f, _)| f.is_script()) + .filter(|(_, m)| m.name == v.1.as_str()) + .map(|(f, _)| f) + .next() + { + // Embedded environment for scripted function + let env = func + .get_shared_encapsulated_environ() + .cloned() + .unwrap_or_else(|| { + // Create a new environment with the current module + crate::Shared::new((&*global).into()) + }); + + let val: Dynamic = crate::FnPtr { + name: v.1.clone(), + curry: <_>::default(), + env: Some(env), + typ: crate::types::fn_ptr::FnPtrType::Script( + func.get_script_fn_def().cloned().unwrap(), + ), + } + .into(); + return Ok(val.into()); + } + } + let index = match expr { // Check if the variable is `this` Expr::ThisPtr(..) => unreachable!("Expr::ThisPtr should have been handled outside"), @@ -66,38 +114,6 @@ impl Engine { #[cfg(not(feature = "no_module"))] debug_assert!(v.2.is_empty(), "variable should not be namespace-qualified"); - // Scripted function with the same name - #[cfg(not(feature = "no_function"))] - if let Some(func) = global - .lib - .iter() - .flat_map(|m| m.iter_fn()) - .filter(|(f, _)| f.is_script()) - .filter(|(_, m)| m.name == v.1.as_str()) - .map(|(f, _)| f) - .next() - { - // Embedded environment for scripted function - let env = func - .get_shared_encapsulated_environ() - .cloned() - .unwrap_or_else(|| { - // Create a new environment with the current module - crate::Shared::new((&*global).into()) - }); - - let val: Dynamic = crate::FnPtr { - name: v.1.clone(), - curry: <_>::default(), - env: Some(env), - typ: crate::types::fn_ptr::FnPtrType::Script( - func.get_script_fn_def().cloned().unwrap(), - ), - } - .into(); - return Ok(val.into()); - } - v.0.map_or(0, NonZeroUsize::get) } diff --git a/tests/fn_ptr.rs b/tests/fn_ptr.rs index 46a7ddda3..a03e75d60 100644 --- a/tests/fn_ptr.rs +++ b/tests/fn_ptr.rs @@ -150,6 +150,29 @@ fn test_fn_ptr_call() { assert_eq!(result, 42); } +/// A bare function name is a function pointer, and stays one after something +/// has set `always_search_scope`. +/// +/// That flag means "do not trust the parse-time variable indices". A name that +/// resolves to a function is not a variable and has no index to distrust, but +/// the check for one used to sit behind the flag — so an `eval` that changed +/// the scope made every later use of a bare function name report it as an +/// unknown variable. +#[test] +#[cfg(not(feature = "no_function"))] +fn test_fn_ptr_from_bare_name_survives_a_scope_change() { + let engine = Engine::new(); + + // The plain case, which always worked. + assert_eq!(engine.eval::("fn dbl(x) { x * 2 } let f = dbl; f.call(4)").unwrap(), 8); + + // And the same once `eval` has changed the scope. + assert_eq!(engine.eval::(r#"fn dbl(x) { x * 2 } eval("let m = 2;"); let f = dbl; f.call(4)"#).unwrap(), 8,); + + // A variable of the same name still wins, which is what the flag is for. + assert_eq!(engine.eval::(r#"fn dbl(x) { x * 2 } eval("let m = 2;"); let dbl = 7; dbl"#).unwrap(), 7,); +} + #[test] #[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_object"))] diff --git a/tests/grain/callback.rs b/tests/grain/callback.rs index 2746bfc29..a8b33fa89 100644 --- a/tests/grain/callback.rs +++ b/tests/grain/callback.rs @@ -58,6 +58,21 @@ fn a_native_can_call_a_named_function_back() { agree("fn double(x) { x * 2 } let a = [1, 2, 3]; a.map(Fn(\"double\"))"); } +/// A bare function name is a function pointer, not a variable read. +/// +/// The compiler leaves it to rhai — it is a fragment — and rhai used to refuse +/// it, because a program holding any fragment sets `always_search_scope` and +/// the check for a function of that name sat behind the flag. Every spelling +/// below reported `double` as an unknown variable. +#[test] +fn a_bare_function_name_is_a_pointer() { + agree("fn double(x) { x * 2 } let a = [1, 2, 3]; a.map(double)"); + agree("fn double(x) { x * 2 } let r = 0; { let f = double; r = f.call(4); } r"); + agree("fn double(x) { x * 2 } fn apply(f, v) { f.call(v) } apply(double, 4)"); + // A variable of the same name still wins over the function. + agree("fn double(x) { x * 2 } let double = 7; double"); +} + /// A capture arrives, and has the right value. /// /// Multiplication commutes, so this says nothing about which *position* it From d005dafb22bc528614079996955295bebfc1e0fd Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 16:19:46 -0700 Subject: [PATCH 31/33] answer type_of and is_shared when called by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are implemented by rhai rather than registered anywhere, and the implementation sat behind `hashes.is_native_only()` inside exec_fn_call. But a call by name decides native_only from the name being *reserved* — and every function answered this way is reserved — so both _call_fn_raw implementations returned early into exec_native_fn_call, straight past the only code that could answer them. type_of then found nothing registered and reported itself missing. Extracted as Engine::exec_syntactic_fn_call and consulted from all three callers, rather than routing the native-only branch through exec_fn_call: that would also have added ensure_no_data_race and a second global.level increment, neither of which belongs in this change. In stock rhai a native calling context.call_fn_raw("type_of", ..) got ErrorFunctionNotFound while the script spelling worked. In grain every call goes that way, so type_of was broken for every non-constant value — `let x = 1; type_of(x)` failed while `type_of(1)` passed only because the optimizer folds it. Five corpus cases and one native-API test. Co-Authored-By: Claude Opus 5 (1M context) --- src/eval/eval_context.rs | 8 ++++ src/func/call.rs | 79 ++++++++++++++++++++++++++------------- src/func/native.rs | 12 ++++++ tests/grain/corpus/mod.rs | 8 ++++ tests/native.rs | 23 ++++++++++++ 5 files changed, 105 insertions(+), 25 deletions(-) diff --git a/src/eval/eval_context.rs b/src/eval/eval_context.rs index ee7e86d1f..fc9eadf4f 100644 --- a/src/eval/eval_context.rs +++ b/src/eval/eval_context.rs @@ -489,6 +489,14 @@ fn _call_fn_raw( let args_len = args.len(); if native_only { + // The functions rhai answers by name are all reserved names, and a + // reserved name is what makes a call native-only — so without this the + // branch routes straight past their only implementation. See + // `Engine::exec_syntactic_fn_call`. + if let Some(result) = engine.exec_syntactic_fn_call(fn_name, args, Position::NONE) { + return result; + } + let hash = calc_fn_hash(None, fn_name, args_len); return engine diff --git a/src/func/call.rs b/src/func/call.rs index 9e1ac16d7..013607bef 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -546,6 +546,57 @@ impl Engine { } } + /// The functions answered by name rather than by dispatch. + /// + /// `type_of` and `is_shared` have no registered implementation anywhere — + /// this *is* their implementation, and the rest of the names here exist only + /// as syntax, so reaching one of them by dispatch is always an error. + /// + /// `None` means the name is not one of these and should be dispatched + /// normally. `Some` is the whole answer: the value, or the error for a + /// spelling that has none — `type_of` with the wrong number of arguments, or + /// a name that is only ever syntax. + /// + /// Shared with [`NativeCallContext::call_fn_raw`], which reaches names it + /// decided were native-only without going through [`Self::exec_fn_call`] at + /// all. Keeping one copy is the point: these used to be answerable only + /// through the script path, so a host calling `call_fn_raw("type_of", ..)` + /// — and the bytecode VM, which dispatches every call that way — got + /// `ErrorFunctionNotFound` for a function rhai does implement. + pub(crate) fn exec_syntactic_fn_call( + &self, + fn_name: &str, + args: &FnCallArgs, + pos: Position, + ) -> Option { + let only_syntax = match fn_name { + // Handle type_of() + KEYWORD_TYPE_OF if args.len() == 1 => { + let typ = self.get_interned_string(self.map_type_name(args[0].type_name())); + return Some(Ok(typ.into())); + } + + #[cfg(not(feature = "no_closure"))] + crate::engine::KEYWORD_IS_SHARED if args.len() == 1 => { + return Some(Ok(args[0].is_shared().into())) + } + #[cfg(not(feature = "no_closure"))] + crate::engine::KEYWORD_IS_SHARED => true, + + #[cfg(not(feature = "no_function"))] + crate::engine::KEYWORD_IS_DEF_FN => true, + + KEYWORD_TYPE_OF | KEYWORD_FN_PTR | KEYWORD_EVAL | KEYWORD_IS_DEF_VAR + | KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY => true, + + _ => false, + }; + + only_syntax.then(|| { + Err(ERR::ErrorFunctionNotFound(self.gen_fn_call_signature(fn_name, args), pos).into()) + }) + } + /// # Main Entry-Point (By Name) /// /// Perform an actual function call, native Rust or scripted, by name, taking care of special functions. @@ -570,32 +621,10 @@ impl Engine { pos: Position, ) -> RhaiResultOf<(Dynamic, bool)> { // These may be redirected from method style calls. - if hashes.is_native_only() - && match fn_name { - // Handle type_of() - KEYWORD_TYPE_OF if args.len() == 1 => { - let typ = self.get_interned_string(self.map_type_name(args[0].type_name())); - return Ok((typ.into(), false)); - } - - #[cfg(not(feature = "no_closure"))] - crate::engine::KEYWORD_IS_SHARED if args.len() == 1 => { - return Ok((args[0].is_shared().into(), false)) - } - #[cfg(not(feature = "no_closure"))] - crate::engine::KEYWORD_IS_SHARED => true, - - #[cfg(not(feature = "no_function"))] - crate::engine::KEYWORD_IS_DEF_FN => true, - - KEYWORD_TYPE_OF | KEYWORD_FN_PTR | KEYWORD_EVAL | KEYWORD_IS_DEF_VAR - | KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY => true, - - _ => false, + if hashes.is_native_only() { + if let Some(result) = self.exec_syntactic_fn_call(fn_name, args, pos) { + return result.map(|value| (value, false)); } - { - let sig = self.gen_fn_call_signature(fn_name, args); - return Err(ERR::ErrorFunctionNotFound(sig, pos).into()); } // Check for data race. diff --git a/src/func/native.rs b/src/func/native.rs index 99e670b58..13511227e 100644 --- a/src/func/native.rs +++ b/src/func/native.rs @@ -523,6 +523,18 @@ impl<'a> NativeCallContext<'a> { let args_len = args.len(); if native_only { + // A reserved name is native-only *because* it is reserved, and the + // ones rhai answers by name rather than by dispatch are all + // reserved — so this branch would otherwise route straight past + // their only implementation. `type_of` has none to find afterwards, + // and reported itself as an unknown function. + if let Some(result) = + self.engine() + .exec_syntactic_fn_call(fn_name, args, self.call_position()) + { + return result; + } + return self .engine() .exec_native_fn_call( diff --git a/tests/grain/corpus/mod.rs b/tests/grain/corpus/mod.rs index cbc5aa985..fd2eddf2e 100644 --- a/tests/grain/corpus/mod.rs +++ b/tests/grain/corpus/mod.rs @@ -585,4 +585,12 @@ pub const CASES: &[Case] = &[ case("closure_for_each_binds_this", "let t = 0; [1, 2, 3].for_each(|| t += this); t"), // And the argument form, which takes the element as a parameter instead. case("closure_map_takes_an_argument", "[1, 2, 3].map(|x| x * 2)"), + // `type_of` has no registered implementation anywhere — rhai answers it by + // name — so it is reached through the same door every other call is. + // A constant argument is folded by the optimizer and proves nothing. + case("type_of_a_variable", "let x = 1; type_of(x)"), + case("type_of_a_container", "let a = [1]; type_of(a)"), + case("type_of_a_host_type", "let w = widget(1); type_of(w)"), + case("type_of_method_style", "let s = \"a\"; s.type_of()"), + case("type_of_a_pointer", "let r = \"\"; { let f = |x| x; r = type_of(f); } r"), ]; diff --git a/tests/native.rs b/tests/native.rs index e2579482c..a43000c8c 100644 --- a/tests/native.rs +++ b/tests/native.rs @@ -53,3 +53,26 @@ fn test_native_overload() { assert_eq!(engine.eval::(r#"let x = "hello"; let y = "world"; x + y"#).unwrap(), "hello***world"); assert_eq!(engine.eval::(r#"let x = "hello"; let y = (); x + y"#).unwrap(), "hello Foo!"); } + +/// A native asking rhai for a function it answers by name rather than by +/// dispatch. +/// +/// `type_of` and `is_shared` have no registered implementation anywhere, and a +/// call by name for one used to route past the code that implements them — +/// they are reserved names, and being reserved is what makes such a call +/// native-only. So a host could ask for `type_of` and be told there is no such +/// function, while the same question written in a script answered fine. +#[test] +fn test_native_call_fn_raw_reaches_syntactic_functions() { + let mut engine = Engine::new(); + + engine.register_raw_fn("ask_type", [TypeId::of::()], |context, args| { + let mut value = args[0].clone(); + context.call_fn_raw("type_of", false, false, &mut [&mut value]) + }); + + assert_eq!(engine.eval::("let x = 1; ask_type(x)").unwrap(), "i64"); + assert_eq!(engine.eval::("let a = [1]; ask_type(a)").unwrap(), "array"); + // And it still agrees with the script spelling. + assert_eq!(engine.eval::("let a = [1]; ask_type(a)").unwrap(), engine.eval::("let a = [1]; type_of(a)").unwrap(),); +} From a661d2423417e7c6c4da098ceb86e7dd05248eb1 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 16:29:56 -0700 Subject: [PATCH 32/33] fix tests --- tests/fn_ptr.rs | 7 ++++--- tests/native.rs | 13 +++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/fn_ptr.rs b/tests/fn_ptr.rs index a03e75d60..94aa00d11 100644 --- a/tests/fn_ptr.rs +++ b/tests/fn_ptr.rs @@ -163,11 +163,12 @@ fn test_fn_ptr_call() { fn test_fn_ptr_from_bare_name_survives_a_scope_change() { let engine = Engine::new(); - // The plain case, which always worked. - assert_eq!(engine.eval::("fn dbl(x) { x * 2 } let f = dbl; f.call(4)").unwrap(), 8); + // Called function-style, which `no_object` leaves in the language where it + // removes the `f.call(..)` spelling. + assert_eq!(engine.eval::("fn dbl(x) { x * 2 } let f = dbl; call(f, 4)").unwrap(), 8); // And the same once `eval` has changed the scope. - assert_eq!(engine.eval::(r#"fn dbl(x) { x * 2 } eval("let m = 2;"); let f = dbl; f.call(4)"#).unwrap(), 8,); + assert_eq!(engine.eval::(r#"fn dbl(x) { x * 2 } eval("let m = 2;"); let f = dbl; call(f, 4)"#).unwrap(), 8,); // A variable of the same name still wins, which is what the flag is for. assert_eq!(engine.eval::(r#"fn dbl(x) { x * 2 } eval("let m = 2;"); let dbl = 7; dbl"#).unwrap(), 7,); diff --git a/tests/native.rs b/tests/native.rs index a43000c8c..c8e3e4786 100644 --- a/tests/native.rs +++ b/tests/native.rs @@ -71,8 +71,13 @@ fn test_native_call_fn_raw_reaches_syntactic_functions() { context.call_fn_raw("type_of", false, false, &mut [&mut value]) }); - assert_eq!(engine.eval::("let x = 1; ask_type(x)").unwrap(), "i64"); - assert_eq!(engine.eval::("let a = [1]; ask_type(a)").unwrap(), "array"); - // And it still agrees with the script spelling. - assert_eq!(engine.eval::("let a = [1]; ask_type(a)").unwrap(), engine.eval::("let a = [1]; type_of(a)").unwrap(),); + // A string, because its type name is the same on every build — `only_i32` + // and `f32_float` rename the numeric ones, and `no_index` removes arrays. + assert_eq!(engine.eval::(r#"let s = "a"; ask_type(s)"#).unwrap(), "string"); + + // And the answer is the script spelling's, whatever the numeric build is. + assert_eq!( + engine.eval::("let x = 1; ask_type(x)").unwrap(), + engine.eval::("let x = 1; type_of(x)").unwrap(), + ); } From b6d72de69058170a8e31b7469dcc5fc9090188c1 Mon Sep 17 00:00:00 2001 From: Jack Hogan Date: Sat, 8 Aug 2026 16:30:06 -0700 Subject: [PATCH 33/33] cargo fmt --- tests/native.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/native.rs b/tests/native.rs index c8e3e4786..3a07e1029 100644 --- a/tests/native.rs +++ b/tests/native.rs @@ -76,8 +76,5 @@ fn test_native_call_fn_raw_reaches_syntactic_functions() { assert_eq!(engine.eval::(r#"let s = "a"; ask_type(s)"#).unwrap(), "string"); // And the answer is the script spelling's, whatever the numeric build is. - assert_eq!( - engine.eval::("let x = 1; ask_type(x)").unwrap(), - engine.eval::("let x = 1; type_of(x)").unwrap(), - ); + assert_eq!(engine.eval::("let x = 1; ask_type(x)").unwrap(), engine.eval::("let x = 1; type_of(x)").unwrap(),); }