diff --git a/benchmarks/string_receiver_boxing.cjs b/benchmarks/string_receiver_boxing.cjs new file mode 100644 index 0000000000..ac165cf5f4 --- /dev/null +++ b/benchmarks/string_receiver_boxing.cjs @@ -0,0 +1,38 @@ +// #9810: run with Node or compile with Perry. Arguments: receiver length, calls. +// .cjs keeps the receiver-binding controls in sloppy mode on both engines. +const n = Number(process.argv[3] || "20000"); +const receiver = "x".repeat(Number(process.argv[2] || "200")); +function unused(value) { return value + 1; } +function observed(value) { return this.length + value; } +function capture() { return this; } +function strict(value) { "use strict"; return value + 1; } +String.prototype.bench9810 = unused; +const funcs = { unused, observed, strict, capture }; +let sum = 0; +let start = Date.now(); +for (let i = 0; i < n; i++) sum += receiver.bench9810(i); +console.log("method", receiver.length, Date.now() - start, sum); +sum = 0; start = Date.now(); +for (let i = 0; i < n; i++) sum += funcs.unused.call(receiver, i); +console.log("call", receiver.length, Date.now() - start, sum); +sum = 0; start = Date.now(); +for (let i = 0; i < n; i++) sum += funcs.observed.call(receiver, i); +console.log("call-this", receiver.length, Date.now() - start, sum); +sum = 0; start = Date.now(); +for (let i = 0; i < n; i++) sum += funcs.unused.apply(receiver, [i]); +console.log("apply", receiver.length, Date.now() - start, sum); +sum = 0; start = Date.now(); +for (let i = 0; i < n; i++) sum += funcs.strict.call(receiver, i); +console.log("strict", receiver.length, Date.now() - start, sum); +sum = 0; start = Date.now(); +for (let i = 0; i < n; i++) sum += Object(receiver).length; +console.log("Object", receiver.length, Date.now() - start, sum); + +const first = funcs.capture.call(receiver); +const second = funcs.capture.apply(receiver, []); +if (typeof first !== "object" || first === second || first.length !== receiver.length) { + throw new Error("sloppy calls must create distinct String wrappers"); +} +first.extra = 7; +if (second.extra !== undefined) throw new Error("receiver state leaked"); +delete String.prototype.bench9810; diff --git a/changelog.d/9755-gc-side-table-young-logs.md b/changelog.d/9755-gc-side-table-young-logs.md index 5fdf78524d..8ac58f2588 100644 --- a/changelog.d/9755-gc-side-table-young-logs.md +++ b/changelog.d/9755-gc-side-table-young-logs.md @@ -3,9 +3,9 @@ - **Minor collections no longer walk every runtime side table.** A copying minor's three root-scan passes — and a budgeted minor's initial root scan and final remark — visited every entry of the closure dynamic-prop tables, - the string-keyed descriptor tables, the shape family/slot-index maps, the - transition cache and the shape cache on every collection, to discover that - nothing in them pointed at the nursery. On the compiled claude-code TUI + the string-keyed descriptor tables, the shape family/slot-index maps and + the transition cache on every collection, to discover that nothing in them + pointed at the nursery. On the compiled claude-code TUI that was ~35k shape families, ~120k descriptors and ~13k closure owners per walk, 41 minors per streamed reply, all reporting `slots=0`: 34–56 ms of scanner time per minor. @@ -26,6 +26,13 @@ prints `[gc-young-log]` rows (logged / visited / kept / table size) per table and cycle. + The **shape cache** was measured and deliberately left on its plain walk. + Its canonical keys arrays are allocated in the longlived arena, which + `addr_is_minor_relevant` must answer `true` for, so no entry ever leaves a + log there: on the claude-code TUI the log named 100 % of the table in every + one of 107 collections (0 % skipped) and cost **35 % more** than the walk it + replaced. The four tables above skip 75–93 %. + - **The post-minor remembered-set coverage restore is proportional to what the dirty scan could not cover.** `restore_surviving_dirty_coverage` (#5029) re-walked every slot of every object on the pre-cycle dirty pages diff --git a/changelog.d/9780-bun-tcp-landing-followups.md b/changelog.d/9780-bun-tcp-landing-followups.md new file mode 100644 index 0000000000..e0293d2f4e --- /dev/null +++ b/changelog.d/9780-bun-tcp-landing-followups.md @@ -0,0 +1,15 @@ +**`Bun.listen` / `Bun.connect` are documented, and `perry-ext-http`'s unit +tests link again.** #9514's TCP socket facades added both methods to the +compile-time API manifest without regenerating the derived docs, so +`docs/src/api/reference.md` and `docs/api/perry.d.ts` had been stale since +2026-09-03 and the API-docs drift check failed on every run. They now list +both entries (3046 → 3048). + +Separately, `js_bun_tcp_listen` drives the shared async runtime from its +bind-poll loop via `perry_ffi::run_pending`. `perry-ext-net` stubs that +symbol only under `#[cfg(test)]`, which does not apply when it is linked as +an ordinary dependency into `perry-ext-http`'s test binary, so release-linking +that crate failed with `undefined symbol: perry_ffi_run_pending`. +`perry-ext-http`'s test shim now provides it, alongside the +`perry_ffi_spawn_async` stub that exists for the same transitive reason. +`perry-ext-ws` was checked and does not need it. diff --git a/changelog.d/9794-alloc-primitive-string-path.md b/changelog.d/9794-alloc-primitive-string-path.md new file mode 100644 index 0000000000..d5b377ae50 --- /dev/null +++ b/changelog.d/9794-alloc-primitive-string-path.md @@ -0,0 +1,29 @@ +### Runtime + +- perf(string): a one-ASCII-character string is now the canonical per-thread + header instead of a fresh 32-byte allocation. `js_string_char_at` — and + everything that funnels through it (`s[i]`, `charAt`, string spread, the + String-wrapper index installer) — used to mint one string per character read, + which on a text-measuring workload is the single largest source of garbage. + Same residency contract as the existing small-integer string table + (longlived arena, `refcount = 0`, pinned, scanned by the same root scanner — + no new scanner is registered). + +- perf(runtime): `String`/`Number`/`Boolean`/`BigInt` wrapper dispatch, + `x.constructor`, `toString` resolution and the `globalThis` builtin lookup + resolve their constant property names through the intern table instead of + minting a heap string per lookup. `js_get_global_this_builtin_value` alone + allocated 133 MB during a 3300-character claude-code reply, all of it the + same handful of literals. Interned keys also make the property-read and + property-write fast paths eligible, which a freshly minted key never was. + +- perf(runtime): a `String` wrapper no longer stores one property descriptor + per character. ECMA-262 §10.4.3 gives every in-range index of a String + exotic object `{ writable: false, enumerable: true, configurable: false }` — + a fact of the class and the boxed length, not per-object state — so + `get_property_attrs` answers it from the wrapper's payload. Storing it cost, + per boxed character, a Rust `String`, a hash-map entry only a full + collection could reclaim, an owner-index entry, and one program-wide + `prop_plan_epoch_bump()`. A sloppy method call on a string primitive boxes + its receiver, so the compiled claude-code TUI paid that for every rendered + line. diff --git a/changelog.d/9794-gc-churn-attribution-diag.md b/changelog.d/9794-gc-churn-attribution-diag.md new file mode 100644 index 0000000000..f4a6a2785d --- /dev/null +++ b/changelog.d/9794-gc-churn-attribution-diag.md @@ -0,0 +1,22 @@ +### Runtime + +- `PERRY_GC_DIAG=1` now says WHY the collector ran, not only what it did: + `[gc-trigger]` prints every predicate input at each collection decision + (armed arena trigger vs `arena_total`, from-space vs the nursery cap, + old-gen reclaimable pressure vs baseline/band, the malloc pair, the + pending/retaining flags); `[gc-full]` names the arm behind every full + mark-sweep with a per-site count; `[gc-budgeted] start/done` reports each + incremental cycle's steps, per-phase step time and root-scan share; + `[gc-charge]` attributes mutator-assist and synchronous-full time to the + calling site (return-address chain resolved to the JS display name); + `[gc-survival]` gives, per copying minor, which root first reached each + surviving byte — shadow stack, native stack map, a named side-table + scanner, or the remembered set split by the old parent's type — with + transitive reach charged to the originating root. +- `PERRY_ALLOC_SITE_SAMPLE=` (arena/alloc_sample.rs): byte-proportional + allocation-site sampling for the GC arena, covering the runtime allocators + and the codegen inline bump path (the mirrored inline block limit is capped + at one interval while sampling). `[alloc-site]` reports bytes by object type + and the top sites after each copying minor and at exit. Off by default; one + relaxed atomic load per allocation when off; the OFF state and the magnitude + parse are pinned in `gc/tests/env_knob_parse.rs`. diff --git a/changelog.d/9796-regex-backtracking-cliff.md b/changelog.d/9796-regex-backtracking-cliff.md new file mode 100644 index 0000000000..f1b388b8e8 --- /dev/null +++ b/changelog.d/9796-regex-backtracking-cliff.md @@ -0,0 +1,43 @@ +### Performance + +- **A capture group no longer turns a pattern into a ReDoS.** + `repeat_matcher::capture_layout` takes a pattern off the linear `regex` + engine when ECMA-262's RepeatMatcher capture semantics are observable — a + capture group directly under a quantifier, or a capture inside a negative + lookaround. That routing is a correctness requirement (the linear engine + keeps the last value of a capture nested in a quantified group; the spec + clears it on every iteration), but the engine it routes to, `regress`, is a + classical backtracker with no step budget. So adding parentheses was enough + to fall off a linear-time path onto an exponential one: + + | pattern | node | perry (before) | perry (after) | + |---|---|---|---| + | `/^(a+)+$/.test("a"×28 + "!")` | 4,798 ms | **16,522 ms** | **0 ms** | + | `/^(?:a+)+$/.test(…)` (same language, no capture) | 4,288 ms | 0 ms | 0 ms | + + **6.3 %** of the 4,463 distinct regex literals across seven real bundles + take that route — claude-code 7.1 %, dayjs 25 %, luxon 29 % — including + shapes like `^[a-z][a-z0-9]*(-[a-z0-9]+)*$`. + + The two engines accept exactly the same LANGUAGE for a pattern they both + compile; they disagree only about which capture assignment to report. So the + linear program is asked first (`linear_rules_out_match`), and when it proves + there is no match at or after the search offset — which is what every ReDoS + input is, a subject that ALMOST matches and then fails — the backtracker is + never entered. Every `&str`-subject entry point goes through + `lookup_repeat_matcher_for`: `test`, `exec`, `match`, `matchAll`, `search`, + `split` and `replace` with a string replacement. The gate disables itself + where the linear engine has no opinion (a pattern it could not compile holds + the never-match placeholder), which is exactly the lookaround shapes. + + **This removes the reachable exponential case; it does not BOUND the worst + case.** A real step budget has to be counted by the backtracker, and + `regress` has none today (`fancy-regex`, by contrast, ships + `backtrack_limit: 1_000_000`). A 101-line patch adding one has been measured + — worst hostile search 51 s → 124 ms at a budget of 1,000,000, zero answers + changed across 13,389 real searches, upstream's own 544 tests unchanged — and + is open upstream as + [ridiculousfish/regress#177](https://github.com/ridiculousfish/regress/pull/177). + Until it lands and perry picks it up, do not read "cliff fixed" as "worst + case bounded". + (`quantified_capture_pattern_does_not_backtrack_on_a_non_matching_subject`) diff --git a/changelog.d/9796-regex-borrowed-cache-keys.md b/changelog.d/9796-regex-borrowed-cache-keys.md new file mode 100644 index 0000000000..76d06c09fa --- /dev/null +++ b/changelog.d/9796-regex-borrowed-cache-keys.md @@ -0,0 +1,26 @@ +### Performance + +- **Probing the compiled-program caches no longer materialises the key.** The + three thread-local caches were `HashMap<(String, String), _>`, and + `HashMap::get` needs a `&(String, String)` — so **every probe allocated two + Strings and copied the pattern text into them**, on a path that runs once per + RegExp OBJECT, and a JS regex literal evaluates to a fresh object every time + it is reached. A native-churn census of the claude-code binary (2026-09-05) + put `js_regexp_test` → `lookup_repeat_matcher` → `build_and_install_programs` + at **6,044 MB of 8,334 MB of estimated allocation with zero live bytes** — + 73 % of all remaining native churn — split across the three probe sites: the + `get_or_compile_regex` probe (2,071 MB) and two `core::fmt::Formatter::pad` + frames (1,989 MB and 1,984 MB), which is what `.to_string()` on an `Arc` + lowers to. + + The caches are now keyed by `ProgramKey = (Arc, Arc)`. Every caller + that matters already holds those `Arc`s — `REGEX_SOURCE_TABLE` and + `regex::site_cache` share one allocation of a literal's text with every + header built from it — so a probe is two refcount increments and no + allocation at all. The two remaining `Arc::from` materialisations are on cold + paths: the syntax-error fallback in `js_regexp_new` (a pattern the linear + engine's parser refused, 7.7 % of real literals, once each) and + `RegExp.prototype.compile` (once per call from user code). + + Hashing still walks the pattern bytes; the allocation is what the census + measured and what this removes. diff --git a/changelog.d/9796-regex-engine-prototype-switch.md b/changelog.d/9796-regex-engine-prototype-switch.md new file mode 100644 index 0000000000..2d3e4f8a68 --- /dev/null +++ b/changelog.d/9796-regex-engine-prototype-switch.md @@ -0,0 +1,31 @@ +### Internal + +- **`PERRY_REGEX_ENGINE=regress` — a measurable tier-0 engine prototype.** + Routes every pattern through `regress` (the ECMAScript backtracker perry + already links for RepeatMatcher capture semantics) instead of only the ones + whose capture semantics require it, and installs a shared never-match + placeholder as the standard program so no NFA is built. Every exec-family + entry point already consults the repeat matcher first, so this exercises the + whole engine surface — `exec`, `test`, `match`, `matchAll`, `search`, + `split`, `replace` — without a second implementation. + + It exists so the engine question is settled on measurements from a real + binary rather than on a corpus harness. Measured over 4,463 distinct regex + literals extracted from seven real bundles (two claude-code builds, ethers, + moment, dayjs, luxon, mongodb) with a tracking allocator and the programs + held live: + + | engine | accepted | compile µs (med) | bytes/program (med) | corpus total | + |---|---|---|---|---| + | `regex` crate (tier 1 today) | 92.3 % | 48.5 | 12,492 | 136.7 MB | + | `regress` | **100 %** | **2.2** | **512** | **4.9 MB** | + | `fancy-regex` (tier 2 today) | 97.8 % | 59.2 | 12,623 | 146.6 MB | + + node/V8, measured the same session, is ~2,600 bytes per program. A + differential over 4,119 patterns × 13 subjects (53,547 comparisons of match + presence, span and every capture span) found **0 disagreements** between the + linear engine and `regress`. + + **Not a supported configuration**: the backtracker has no step budget, so a + pathological pattern can run unbounded. Off by default, one relaxed atomic + load when unset. diff --git a/changelog.d/9814-virtual-string-indices.md b/changelog.d/9814-virtual-string-indices.md new file mode 100644 index 0000000000..8a063471e8 --- /dev/null +++ b/changelog.d/9814-virtual-string-indices.md @@ -0,0 +1 @@ +Fix string-wrapper construction scaling with receiver length in non-strict method calls, `Function.prototype.call`/`apply`, and `Object(string)`. Character indices are now virtual properties, preserving UTF-16 indexing, reflection, and readonly descriptors without allocating a property and descriptor for every character (#9810). diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index f6a8e47d99..bb860ad6e7 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -524,7 +524,7 @@ pub(crate) fn inline_hot_small_max_call_sites() -> u32 { /// (every function stays on native statepoints, the pre-#8583 behavior). const DEFAULT_ROOT_SPILL_RELOCATIONS: usize = 32_000_000; -fn root_spill_relocation_threshold() -> usize { +pub(crate) fn root_spill_relocation_threshold() -> usize { std::env::var("PERRY_ROOT_SPILL_RELOCATIONS") .ok() .and_then(|v| v.trim().parse::().ok()) diff --git a/crates/perry-codegen/src/codegen/mod.rs b/crates/perry-codegen/src/codegen/mod.rs index dc7c2b43b9..d3d84cb5a1 100644 --- a/crates/perry-codegen/src/codegen/mod.rs +++ b/crates/perry-codegen/src/codegen/mod.rs @@ -386,6 +386,18 @@ pub fn short_spread_method_capabilities(hir: &HirModule) -> Vec String { + let module_prefix = sanitize(module_name); + helpers::scoped_fn_name(&module_prefix, function_name) +} + /// Compile a Perry HIR module to an object file via LLVM IR. /// /// CRITICAL (#686): `hir` MUST be `&HirModule` (shared reference), never diff --git a/crates/perry-codegen/src/expr/dyn_extern_i18n.rs b/crates/perry-codegen/src/expr/dyn_extern_i18n.rs index 7097bc3868..d193041602 100644 --- a/crates/perry-codegen/src/expr/dyn_extern_i18n.rs +++ b/crates/perry-codegen/src/expr/dyn_extern_i18n.rs @@ -489,8 +489,15 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { is_eval: _, } => { let _ = lower_expr(ctx, filename)?; + if ctx.block().is_terminated() { + return Ok(double_literal(f64::from_bits(crate::nanbox::TAG_UNDEFINED))); + } let options_val = if let Some(options) = options { - lower_expr(ctx, options)? + let value = lower_expr(ctx, options)?; + if ctx.block().is_terminated() { + return Ok(double_literal(f64::from_bits(crate::nanbox::TAG_UNDEFINED))); + } + value } else { double_literal(f64::from_bits(crate::nanbox::TAG_UNDEFINED)) }; diff --git a/crates/perry-codegen/src/inprocess.rs b/crates/perry-codegen/src/inprocess.rs index 5b83456dc5..7b4affa613 100644 --- a/crates/perry-codegen/src/inprocess.rs +++ b/crates/perry-codegen/src/inprocess.rs @@ -17,6 +17,9 @@ //! IR and flags this pipeline produces objects byte-identical to Homebrew //! clang 22's `clang -c`. +mod optimize_emit; +use optimize_emit::optimize_and_emit; + use std::ffi::CString; use std::sync::Once; @@ -308,7 +311,8 @@ pub(crate) fn optimize_and_emit_module( } /// [`optimize_and_emit_module`] that also fills `stats` (sizes before and -/// after RS4GC, widest functions, phase times) for the per-unit report. +/// after RS4GC, widest functions, phase times, and any bounded-emission +/// fallback) for the per-unit report. pub(crate) fn optimize_and_emit_module_with_stats( module: &inkwell::module::Module<'_>, effective_target: &str, @@ -348,6 +352,9 @@ pub struct UnitCodegenStats { /// Functions stamped `"disable-tail-calls"` because their alloca-walk /// estimate exceeded [`DEFAULT_TRE_MAX_ALLOCA_WALK`] (#8883). pub tail_call_elim_skipped: Vec, + /// The widest function which made this unit use LLVM's bounded O0 machine + /// pipeline after completing the requested IR optimization pipeline. + pub fast_emit_fallback: Option, } fn function_instruction_count(function: inkwell::values::FunctionValue<'_>) -> usize { @@ -385,6 +392,133 @@ fn module_instruction_census( (functions, total, widest) } +/// Per-function instruction ceiling for LLVM's optimized machine pipeline. +/// +/// This budget is checked *after* the requested `default` IR pipeline has +/// completed. It changes neither JS lowering nor middle-end optimization; it +/// only asks the target machine to use its O0 instruction-selection, +/// live-interval and register-allocation pipeline for a unit containing an +/// extreme generated function. +/// +/// The threshold is bracketed by real arm64/LLVM 22 measurements. Machine-IR +/// expansion depends on CFG shape, so raw IR size is deliberately only a +/// conservative guard: one 161k-instruction function emitted normally in +/// ~19s, while a different 100,152-instruction Claude Code 2.1.259 function +/// grew past ~10 GiB RSS in the optimized machine pipeline. The same function +/// emitted through an O0 target machine in 6s. Another 277k-instruction async +/// state-machine function remained in LiveIntervals / register allocation for +/// more than 16 minutes at ~10 GiB RSS; its already-Os-optimized IR emitted +/// through an O0 target machine in 3.5s at ~550 MiB RSS. 100k is immediately +/// below the smallest observed pathological case. +/// +/// `PERRY_LL_FAST_EMIT_MAX_INSTRS=` raises or lowers the ceiling; `0` / +/// `off` disables the fallback. +const DEFAULT_FAST_EMIT_MAX_INSTRS: usize = 100_000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FastEmitBudget { + Off, + Cap(usize), +} + +fn parse_fast_emit_budget(value: Option<&str>) -> FastEmitBudget { + match value.map(str::trim) { + None | Some("") => FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS), + Some("0") | Some("off") | Some("false") => FastEmitBudget::Off, + Some(v) => match v.parse::() { + Ok(0) => FastEmitBudget::Off, + Ok(n) => FastEmitBudget::Cap(n), + Err(_) => FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS), + }, + } +} + +fn fast_emit_budget() -> FastEmitBudget { + #[cfg(test)] + if let Some(budget) = TEST_FAST_EMIT_BUDGET.with(std::cell::Cell::get) { + return budget; + } + parse_fast_emit_budget( + std::env::var("PERRY_LL_FAST_EMIT_MAX_INSTRS") + .ok() + .as_deref(), + ) +} + +#[cfg(test)] +thread_local! { + static TEST_FAST_EMIT_BUDGET: std::cell::Cell> = const { + std::cell::Cell::new(None) + }; +} + +/// Thread-local budget seam; mutating the process environment would race the +/// other LLVM tests in this binary. +#[cfg(test)] +fn with_test_fast_emit_budget(cap: usize, run: impl FnOnce() -> T) -> T { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + TEST_FAST_EMIT_BUDGET.with(|budget| budget.set(self.0)); + } + } + let old = TEST_FAST_EMIT_BUDGET.replace(Some(FastEmitBudget::Cap(cap))); + let _restore = Restore(old); + run() +} + +/// The extreme function which selected bounded machine-code emission. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FastEmitFallback { + pub name: String, + pub instructions: usize, + pub cap: usize, +} + +impl std::fmt::Display for FastEmitFallback { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "`{}` has {} instructions after IR optimization, above the optimized machine-pipeline \ + budget {}; keeping the requested IR optimization, then emitting this unit through \ + LLVM's O0 machine pipeline to bound instruction selection, live intervals and \ + register allocation. Override with PERRY_LL_FAST_EMIT_MAX_INSTRS= (raise) or \ + =0 (disable).", + self.name, self.instructions, self.cap + ) + } +} + +fn fast_emit_fallback( + module: &inkwell::module::Module<'_>, + budget: FastEmitBudget, +) -> Option { + let cap = match budget { + FastEmitBudget::Off => return None, + FastEmitBudget::Cap(cap) => cap, + }; + let mut widest: Option = None; + let mut function = module.get_first_function(); + while let Some(f) = function { + if f.count_basic_blocks() > 0 { + let instructions = function_instruction_count(f); + if instructions > cap + && widest + .as_ref() + .is_none_or(|current| instructions > current.instructions) + { + widest = Some(FastEmitFallback { + name: f.get_name().to_string_lossy().into_owned(), + instructions, + cap, + }); + } + } + function = f.get_next_function(); + } + widest +} + /// Instruction budget for ONE function after `rewrite-statepoints-for-gc`. /// /// This is the measured backstop for the estimate that keeps relocation @@ -409,15 +543,31 @@ enum RewriteBudget { /// One function that must be re-lowered onto a shadow frame before LLVM can /// safely optimize its codegen unit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Rs4gcBudgetCause { + /// The constructed function is already large enough that RS4GC's own + /// liveness/rewrite walk may not finish. The estimate uses the roots and + /// non-leaf call sites LLVM will actually see, rather than another source + /// syntax approximation. + PreRewrite { + root_allocas: usize, + safepoints: usize, + estimated_relocations: usize, + }, + /// RS4GC finished, but its relocation fan-out made the rewritten body too + /// large for the normal optimization pipeline. + PostRewrite { post_instructions: usize }, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Rs4gcBudgetViolation { /// LLVM symbol of the function to spill. pub name: String, /// Instruction count before RS4GC, when the caller requested a census. pub pre_instructions: Option, - /// Instruction count after RS4GC and before the optimizer. - pub post_instructions: usize, - /// Active per-function instruction limit. + /// The pre- or post-rewrite condition that requested the retry. + pub cause: Rs4gcBudgetCause, + /// Active limit for the cause's estimate. pub cap: usize, } @@ -547,6 +697,120 @@ fn rs4gc_functions(module: &inkwell::module::Module<'_>) -> std::collections::Ha names } +/// The two constructed-IR factors that bound RS4GC relocation fan-out. +/// +/// Count only allocas whose payload is a managed pointer and call sites which +/// are not explicitly marked as GC leaves. LLVM intrinsics are also leaves: +/// they cannot enter Perry's runtime or collect. This is deliberately the +/// same conservative model as the source-level spill estimate — each +/// safepoint can leave one additional pointer result live across later calls — +/// but it observes the calls codegen actually emitted. That closes estimator +/// holes where one source expression expands into several collecting helpers. +fn rs4gc_preflight_factors(function: inkwell::values::FunctionValue<'_>) -> (usize, usize) { + let mut root_allocas = 0usize; + let mut safepoints = 0usize; + for bb in function.get_basic_blocks() { + let mut inst = bb.get_first_instruction(); + while let Some(i) = inst { + match i.get_opcode() { + inkwell::values::InstructionOpcode::Alloca => { + if matches!( + i.get_allocated_type(), + Ok(inkwell::types::BasicTypeEnum::PointerType(ptr)) + if ptr.get_address_space() == inkwell::AddressSpace::from(1u16) + ) { + root_allocas += 1; + } + } + inkwell::values::InstructionOpcode::Call + | inkwell::values::InstructionOpcode::CallBr + | inkwell::values::InstructionOpcode::Invoke => { + // Call, invoke and callbr are all LLVM CallBase values, so + // the call-site attribute API is valid for each opcode. + let call = unsafe { inkwell::values::CallSiteValue::new(i.as_value_ref()) }; + let gc_leaf = call + .get_string_attribute( + inkwell::attributes::AttributeLoc::Function, + "gc-leaf-function", + ) + .is_some(); + let intrinsic = call + .get_called_fn_value() + .map_or(false, |callee| callee.get_intrinsic_id() != 0); + if !gc_leaf && !intrinsic { + safepoints += 1; + } + } + _ => {} + } + inst = i.get_next_instruction(); + } + } + (root_allocas, safepoints) +} + +/// Every RS4GC-participating function whose constructed IR predicts more +/// relocation work than the source-level spill budget permits. +fn rs4gc_preflight_violations( + module: &inkwell::module::Module<'_>, + cap: usize, + rewritten_functions: &std::collections::HashSet, +) -> Vec<(String, usize, usize, usize)> { + if cap == 0 { + return Vec::new(); + } + let mut over = Vec::new(); + let mut function = module.get_first_function(); + while let Some(f) = function { + if f.count_basic_blocks() > 0 { + let name = f.get_name().to_string_lossy().into_owned(); + if rewritten_functions.contains(&name) { + let (root_allocas, safepoints) = rs4gc_preflight_factors(f); + let live_roots = + crate::codegen::helpers::spill_live_root_count(root_allocas, safepoints); + let estimate = + crate::codegen::helpers::root_relocation_estimate(live_roots, safepoints); + if estimate > cap { + over.push((name, root_allocas, safepoints, estimate)); + } + } + } + function = f.get_next_function(); + } + over +} + +/// Stop before RS4GC itself enters its super-linear liveness/rewrite walk and +/// ask codegen to re-lower the named functions with precise shadow roots. +fn enforce_rs4gc_preflight_budget( + module: &inkwell::module::Module<'_>, + cap: usize, + pre: &std::collections::HashMap, + rewritten_functions: &std::collections::HashSet, +) -> Result<()> { + let violations: Vec = + rs4gc_preflight_violations(module, cap, rewritten_functions) + .into_iter() + .map( + |(name, root_allocas, safepoints, estimated_relocations)| Rs4gcBudgetViolation { + pre_instructions: pre.get(&name).copied(), + name, + cause: Rs4gcBudgetCause::PreRewrite { + root_allocas, + safepoints, + estimated_relocations, + }, + cap, + }, + ) + .collect(); + if violations.is_empty() { + Ok(()) + } else { + Err(anyhow::Error::new(Rs4gcBudgetExceeded { violations })) + } +} + /// Every RS4GC-participating function whose post-rewrite body exceeds `cap`. fn rs4gc_budget_violations( module: &inkwell::module::Module<'_>, @@ -569,23 +833,40 @@ fn rs4gc_budget_violations( } fn rewrite_budget_message(violation: &Rs4gcBudgetViolation, retry: bool) -> String { - let before = violation - .pre_instructions - .map(|n| format!(" (it was {n} before the rewrite)")) - .unwrap_or_default(); let outcome = if retry { "Perry will re-lower this function with precise roots in a shadow frame, then retry the \ unit at the requested optimization level" } else { "the warning-only budget override leaves the function for LLVM to optimize" }; - format!( - "rewrite-statepoints-for-gc grew `{}` to {} instructions{before}; the \ - per-function budget is {}. LLVM's optimizer is super-linear on statepoint \ - relocation fan-out of this size; {outcome} (#8679). Override with \ - PERRY_LL_RS4GC_MAX_INSTRS= (raise), =warn: (warn only) or =0 (disable).", - violation.name, violation.post_instructions, violation.cap - ) + match &violation.cause { + Rs4gcBudgetCause::PreRewrite { + root_allocas, + safepoints, + estimated_relocations, + } => format!( + "before rewrite-statepoints-for-gc, `{}` has {root_allocas} managed-root allocas and \ + {safepoints} non-leaf call sites; accounting for call-result temporaries predicts \ + {estimated_relocations} relocations, above the pre-rewrite budget {}. RS4GC's own \ + liveness/rewrite walk is super-linear on fan-out of this size; {outcome} (#8583). \ + Override with PERRY_ROOT_SPILL_RELOCATIONS= (raise) or =0 (disable).", + violation.name, violation.cap + ), + Rs4gcBudgetCause::PostRewrite { post_instructions } => { + let before = violation + .pre_instructions + .map(|n| format!(" (it was {n} before the rewrite)")) + .unwrap_or_default(); + format!( + "rewrite-statepoints-for-gc grew `{}` to {post_instructions} \ + instructions{before}; the per-function budget is {}. LLVM's optimizer is \ + super-linear on statepoint relocation fan-out of this size; {outcome} (#8679). \ + Override with PERRY_LL_RS4GC_MAX_INSTRS= (raise), =warn: (warn only) or \ + =0 (disable).", + violation.name, violation.cap + ) + } + } } /// Apply [`RewriteBudget`] to a rewritten module. `pre` gives each function's @@ -610,7 +891,7 @@ fn enforce_rs4gc_instruction_budget( .map(|(name, post_instructions)| Rs4gcBudgetViolation { pre_instructions: pre.get(&name).copied(), name, - post_instructions, + cause: Rs4gcBudgetCause::PostRewrite { post_instructions }, cap, }) .collect(); @@ -825,1065 +1106,3 @@ fn disable_tail_call_elim_over_budget<'ctx>( } over } - -fn optimize_and_emit( - module: &inkwell::module::Module<'_>, - effective_target: &str, - opt: char, - mcpu_native: bool, - explicit_cpu: Option<&str>, - mllvm: &[String], - emit_asm: bool, - native_roots: bool, - mut stats: Option<&mut UnitCodegenStats>, -) -> Result> { - global_init(mllvm); - announce(); - - module - .verify() - .map_err(|e| anyhow!("LLVM verifier rejected module:\n{}", e.to_string()))?; - - let triple = TargetTriple::create(effective_target); - let target = Target::from_triple(&triple) - .map_err(|e| anyhow!("no LLVM target for `{effective_target}`: {e}"))?; - let (cpu, features) = if mcpu_native { - ( - TargetMachine::get_host_cpu_name() - .to_string_lossy() - .into_owned(), - TargetMachine::get_host_cpu_features() - .to_string_lossy() - .into_owned(), - ) - } else if let Some(cpu) = explicit_cpu { - (cpu.to_string(), String::new()) - } else { - ( - default_cpu_for_triple(effective_target).to_string(), - String::new(), - ) - }; - let opt_level = match opt { - '0' => OptimizationLevel::None, - '1' => OptimizationLevel::Less, - '2' | 's' | 'z' => OptimizationLevel::Default, - _ => OptimizationLevel::Aggressive, - }; - let tm = target - .create_target_machine( - &triple, - &cpu, - &features, - opt_level, - RelocMode::PIC, - CodeModel::Default, - ) - .ok_or_else(|| anyhow!("failed to create TargetMachine for `{effective_target}`"))?; - - // Same trust order as the subprocess path: `-target` wins over whatever - // triple the module text states, and the module optimizes under the - // machine's real datalayout. - module.set_triple(&triple); - module.set_data_layout(&tm.get_target_data().get_data_layout()); - - // RS4GC must run BEFORE the optimization pipeline, and — critically — in - // this process, against this LLVM. - // - // The external path shells `rewrite-statepoints-for-gc` out to an `opt` - // binary and then hands the rewritten IR to `clang -c`. When those are - // different LLVM versions (Homebrew 22 and Apple clang 21 is the ordinary - // macOS case) the emitted IR uses constructs the older parser rejects, and - // the compile dies with `error: unterminated attribute group`. That is why - // RS4GC needed `PERRY_LLVM_CLANG` pointed at a version-matched toolchain, - // and why it did not work on a stock install at all. - // - // Here the same `TargetMachine` runs the pass and emits the object, so the - // skew cannot exist. This matters beyond convenience: RS4GC is the only - // backend that can root an `invoke`, and since #7302 every call inside a - // `try` is one — 26% of the gap suite (128 of 479 files) contains a `try`, - // which the explicit bridge refuses outright (#7327/#7330). - if native_roots { - // Sizes before the rewrite: the budget message below names them, and - // the per-unit report compares them with the post-rewrite census. - let budget = rs4gc_instruction_budget(); - let rewritten_functions = rs4gc_functions(module); - let pre_sizes = if budget == RewriteBudget::Off && stats.is_none() { - std::collections::HashMap::new() - } else { - pre_rewrite_sizes(module) - }; - if let Some(stats) = stats.as_deref_mut() { - stats.functions = pre_sizes.len(); - stats.pre_rewrite_instructions = pre_sizes.values().sum(); - stats.pre_rewrite_widest = pre_sizes - .iter() - .max_by_key(|(_, n)| **n) - .map(|(name, n)| (name.clone(), *n)); - } - let rewrite_started = std::time::Instant::now(); - module - .run_passes(STATEPOINT_REWRITE_PASSES, &tm, PassBuilderOptions::create()) - .map_err(|e| { - anyhow!( - "in-process rewrite-statepoints-for-gc failed:\n{}", - e.to_string() - ) - })?; - // Verify the rewritten module before it reaches the backend. RS4GC - // has produced verifier-invalid IR in the wild (#8121: it wrapped an - // inline-asm barrier into a gc.statepoint), and unlike the external - // `opt` path — whose verifier aborts with the broken instruction — - // the in-process pipeline would feed the broken module straight to - // ISel, where it dies as a bare SIGBUS with no diagnostic. - module.verify().map_err(|e| { - anyhow!( - "in-process rewrite-statepoints-for-gc produced a module the \ - verifier rejects (this is a Perry codegen bug — the input \ - shape must be exempted or fixed):\n{}", - e.to_string() - ) - })?; - if let Some(stats) = stats.as_deref_mut() { - stats.rewrite_secs = rewrite_started.elapsed().as_secs_f64(); - let (_, total, widest) = module_instruction_census(module); - stats.post_rewrite_instructions = total; - stats.post_rewrite_widest = widest; - } - // The relocation-fan-out backstop (#8583/#8679): stop before the - // super-linear optimizer and ask codegen to retry the named functions - // with precise shadow-frame roots. The retry keeps this same pipeline - // and optimization level; only the GC-root representation changes. - enforce_rs4gc_instruction_budget(module, budget, &pre_sizes, &rewritten_functions)?; - } - - let pipeline = match opt { - '0' => "default", - '1' => "default", - '2' => "default", - 's' => "default", - 'z' => "default", - _ => "default", - }; - // TailCallElim runs inside every `default` function-simplification - // pipeline; bound its alloca walk on the module the pipeline will see - // (#8883). `-O0` runs no TRE, so there is nothing to bound. - if opt != '0' { - let skipped = disable_tail_call_elim_over_budget(module, tre_walk_budget()); - for over in &skipped { - eprintln!("perry: {over}"); - } - if let Some(stats) = stats.as_deref_mut() { - stats.tail_call_elim_skipped = skipped; - } - } - let optimize_started = std::time::Instant::now(); - module - .run_passes(pipeline, &tm, PassBuilderOptions::create()) - .map_err(|e| anyhow!("pass pipeline `{pipeline}` failed:\n{}", e.to_string()))?; - if let Some(stats) = stats.as_deref_mut() { - stats.optimize_secs = optimize_started.elapsed().as_secs_f64(); - } - - let kind = if emit_asm { - FileType::Assembly - } else { - FileType::Object - }; - let emit_started = std::time::Instant::now(); - let obj = tm - .write_to_memory_buffer(module, kind) - .map_err(|e| anyhow!("{kind:?} emission failed:\n{}", e.to_string()))?; - if let Some(stats) = stats { - stats.emit_secs = emit_started.elapsed().as_secs_f64(); - } - Ok(obj.as_slice().to_vec()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn relocation_results(ir: &str) -> std::collections::HashSet<&str> { - ir.lines() - .filter(|line| line.contains("@llvm.experimental.gc.relocate")) - .filter_map(|line| line.trim().split_once(" = ").map(|(result, _)| result)) - .collect() - } - - fn returned_gc_pointers(ir: &str) -> Vec<&str> { - ir.lines() - .filter_map(|line| { - line.trim() - .strip_prefix("ret ptr addrspace(1) ") - .and_then(|value| value.split_whitespace().next()) - }) - .collect() - } - - fn asm_barrier_fixture(leaf_attr: &str) -> String { - format!( - "declare i64 @may_collect()\n\n\ - define i64 @f(i64 %a) gc \"statepoint-example\" {{\n\ - entry:\n\ - \x20 %slot = alloca ptr addrspace(1)\n\ - \x20 %p = inttoptr i64 %a to ptr addrspace(1)\n\ - \x20 store ptr addrspace(1) %p, ptr %slot\n\ - \x20 call void asm sideeffect \"\", \"\"(){leaf_attr}\n\ - \x20 %t = call i64 @may_collect()\n\ - \x20 %after = load ptr addrspace(1), ptr %slot\n\ - \x20 %bits = ptrtoint ptr addrspace(1) %after to i64\n\ - \x20 %r = add i64 %t, %bits\n\ - \x20 ret i64 %r\n\ - }}\n" - ) - } - - #[test] - fn gc_leaf_asm_barrier_survives_rs4gc_unwrapped() { - // The shipped emitters stamp the loop-preservation barrier - // `"gc-leaf-function"`; RS4GC must leave it as a plain inline-asm - // call while still statepointing the real call next to it. - let rewritten = statepoint_rewritten_ir( - &asm_barrier_fixture(" \"gc-leaf-function\""), - "arm64-apple-darwin", - "asm_barrier_leaf", - ) - .expect("attributed barrier must survive the rewrite"); - assert!( - rewritten.contains("call void asm sideeffect"), - "barrier must remain a plain inline-asm call:\n{rewritten}" - ); - assert!( - !rewritten.contains("elementtype(void ()) asm"), - "barrier must not be statepoint-wrapped:\n{rewritten}" - ); - assert!( - rewritten.contains("@llvm.experimental.gc.statepoint"), - "the genuine call must still be statepointed:\n{rewritten}" - ); - } - - #[test] - fn unattributed_asm_barrier_is_rejected_not_miscompiled() { - // Sabotage arm: without the attribute RS4GC wraps the asm into a - // gc.statepoint whose callee is inline asm — invalid IR. The - // pipeline must fail verification loudly (#8121's SIGBUS shape), - // proving the leaf test above can actually fail. - let result = statepoint_rewritten_ir( - &asm_barrier_fixture(""), - "arm64-apple-darwin", - "asm_barrier_broken", - ); - assert!( - result.is_err(), - "an unattributed barrier must be rejected by the verifier" - ); - } - - #[test] - fn rewrite_budget_spellings() { - assert_eq!( - parse_rewrite_budget(None), - RewriteBudget::Error(DEFAULT_RS4GC_MAX_INSTRS) - ); - assert_eq!(parse_rewrite_budget(Some("0")), RewriteBudget::Off); - assert_eq!(parse_rewrite_budget(Some("off")), RewriteBudget::Off); - assert_eq!( - parse_rewrite_budget(Some(" 250000 ")), - RewriteBudget::Error(250_000) - ); - assert_eq!( - parse_rewrite_budget(Some("warn:4096")), - RewriteBudget::Warn(4096) - ); - assert_eq!(parse_rewrite_budget(Some("warn:0")), RewriteBudget::Off); - // Unparsable values keep the default rather than silently disabling. - assert_eq!( - parse_rewrite_budget(Some("lots")), - RewriteBudget::Error(DEFAULT_RS4GC_MAX_INSTRS) - ); - } - - /// Six gc values live across forty safepoints: ~60 instructions before - /// `rewrite-statepoints-for-gc`, a few hundred after (each statepoint - /// relocates every live value). A budget between the two is exceeded - /// only by the post-rewrite module — which is the property the - /// assertion exists for. Counting BEFORE the rewrite (the #8421 - /// replacement knob's mistake) would make `after` empty and fail here. - fn relocation_fanout_fixture() -> String { - let mut ir = String::from( - "declare i64 @may_collect()\n\n\ - define i64 @f(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5) gc \"statepoint-example\" {\n\ - entry:\n", - ); - for i in 0..6 { - ir.push_str(&format!( - " %p{i} = inttoptr i64 %a{i} to ptr addrspace(1)\n" - )); - } - for c in 0..40 { - ir.push_str(&format!(" %c{c} = call i64 @may_collect()\n")); - } - for i in 0..6 { - ir.push_str(&format!( - " %b{i} = ptrtoint ptr addrspace(1) %p{i} to i64\n" - )); - } - ir.push_str( - " %s0 = add i64 %b0, %b1\n %s1 = add i64 %s0, %b2\n %s2 = add i64 %s1, %b3\n\ - \x20 %s3 = add i64 %s2, %b4\n %s4 = add i64 %s3, %b5\n %s5 = add i64 %s4, %c0\n\ - \x20 %s6 = add i64 %s5, %c39\n ret i64 %s6\n}\n", - ); - ir - } - - #[test] - fn rs4gc_budget_fires_only_on_the_rewritten_module() { - global_init(&[]); - let target = "arm64-apple-darwin"; - let fixture = relocation_fanout_fixture(); - let rewritten = statepoint_rewritten_ir(&fixture, target, "fanout_budget") - .expect("fan-out fixture must run RS4GC"); - - let context = Context::create(); - let before = parse_ir_text(&context, &fixture, "fanout_before").expect("fixture parses"); - let after = parse_ir_text(&context, &rewritten, "fanout_after").expect("rewritten parses"); - let pre = pre_rewrite_sizes(&before); - let rewritten_functions = rs4gc_functions(&before); - let pre_f = pre["f"]; - let (_, post_total, post_widest) = module_instruction_census(&after); - let post_f = post_widest.as_ref().map(|(_, n)| *n).unwrap_or(0); - assert!( - post_f > 3 * pre_f, - "fixture must grow under relocation fan-out (pre {pre_f}, post {post_f}):\n{rewritten}" - ); - assert_eq!(post_total, post_f, "one defined function"); - let cap = pre_f + (post_f - pre_f) / 2; - - assert!( - rs4gc_budget_violations(&before, cap, &rewritten_functions).is_empty(), - "the pre-rewrite module is under the budget by construction" - ); - let over = rs4gc_budget_violations(&after, cap, &rewritten_functions); - assert_eq!( - over.len(), - 1, - "exactly the rewritten body is over: {over:?}" - ); - assert_eq!(over[0].0, "f"); - assert_eq!(over[0].1, post_f); - - let err = enforce_rs4gc_instruction_budget( - &after, - RewriteBudget::Error(cap), - &pre, - &rewritten_functions, - ) - .expect_err("the default spelling requests a spill retry"); - let retry = rs4gc_budget_retry(&err).expect("the request stays typed"); - assert_eq!(retry.len(), 1); - assert_eq!(retry[0].name, "f"); - assert_eq!(retry[0].pre_instructions, Some(pre_f)); - assert_eq!(retry[0].post_instructions, post_f); - assert_eq!(retry[0].cap, cap); - let msg = format!("{err:#}"); - for needle in [ - "`f`", - &format!("to {post_f} instructions"), - &format!("it was {pre_f} before"), - &format!("budget is {cap}"), - "PERRY_LL_RS4GC_MAX_INSTRS", - "re-lower", - "#8679", - ] { - assert!( - msg.contains(needle), - "message must carry {needle:?}:\n{msg}" - ); - } - assert!( - !msg.contains("optnone"), - "the budget is an assertion, never a demotion:\n{msg}" - ); - enforce_rs4gc_instruction_budget( - &after, - RewriteBudget::Warn(cap), - &pre, - &rewritten_functions, - ) - .expect("warn spelling does not retry"); - enforce_rs4gc_instruction_budget(&after, RewriteBudget::Off, &pre, &rewritten_functions) - .expect("off spelling does not retry"); - enforce_rs4gc_instruction_budget( - &after, - RewriteBudget::Error(post_f), - &pre, - &rewritten_functions, - ) - .expect("a budget at the exact size is not exceeded"); - - // The retry removes the function's GC strategy. Its ordinary shadow - // body may itself exceed a deliberately tiny test cap, but it must not - // request the same spill forever: only functions that entered RS4GC - // are governed by this relocation-fan-out budget. - let no_rewritten_functions = std::collections::HashSet::new(); - enforce_rs4gc_instruction_budget( - &after, - RewriteBudget::Error(cap), - &pre, - &no_rewritten_functions, - ) - .expect("a shadow-spilled function is outside the RS4GC budget"); - } - - fn constant_fold_order_fixture(folded: bool) -> String { - let mut ir = String::from( - "declare i64 @may_collect()\n\ndefine i64 @f(i64 %d0, i64 %d1, i64 %d2, i64 %d3, i64 %d4, i64 %d5, i64 %d6, i64 %d7) gc \"statepoint-example\" {\nentry:\n", - ); - for i in 0..8 { - ir.push_str(&format!(" %cslot{i} = alloca ptr addrspace(1)\n")); - if folded { - ir.push_str(&format!( - " store ptr addrspace(1) inttoptr (i64 9222246136947933185 to ptr addrspace(1)), ptr %cslot{i}\n" - )); - } else { - ir.push_str(&format!( - " %cb{i} = bitcast double 0x7FFC000000000001 to i64\n %cp{i} = inttoptr i64 %cb{i} to ptr addrspace(1)\n store ptr addrspace(1) %cp{i}, ptr %cslot{i}\n" - )); - } - } - for i in 0..8 { - ir.push_str(&format!( - " %dslot{i} = alloca ptr addrspace(1)\n %dp{i} = inttoptr i64 %d{i} to ptr addrspace(1)\n store ptr addrspace(1) %dp{i}, ptr %dslot{i}\n" - )); - } - ir.push_str(" %sp = call i64 @may_collect()\n"); - for i in 0..8 { - ir.push_str(&format!( - " %after{i} = load ptr addrspace(1), ptr %dslot{i}\n %bits{i} = ptrtoint ptr addrspace(1) %after{i} to i64\n" - )); - } - for i in 0..8 { - ir.push_str(&format!( - " %cafter{i} = load ptr addrspace(1), ptr %cslot{i}\n %cbits{i} = ptrtoint ptr addrspace(1) %cafter{i} to i64\n" - )); - } - ir.push_str(" %x1 = xor i64 %bits0, %bits1\n"); - for i in 2..8 { - ir.push_str(&format!(" %x{i} = xor i64 %x{}, %bits{i}\n", i - 1)); - } - ir.push_str(" %y0 = xor i64 %x7, %cbits0\n"); - for i in 1..8 { - ir.push_str(&format!(" %y{i} = xor i64 %y{}, %cbits{i}\n", i - 1)); - } - ir.push_str(" ret i64 %y7\n}\n"); - ir - } - - #[test] - fn rs4gc_canonicalizes_construction_time_folds_before_root_liveness() { - let _native = crate::codegen::helpers::NativeRootsPin::native(); - let target = crate::codegen::default_target_triple(); - let text_ir = constant_fold_order_fixture(false); - let folded_ir = constant_fold_order_fixture(true); - - for (label, ir) in [("text", &text_ir), ("folded", &folded_ir)] { - let rewritten = statepoint_rewritten_ir(ir, &target, label) - .unwrap_or_else(|e| panic!("{label} fixture must run RS4GC: {e:#}")); - assert!( - !rewritten.contains("%cb0 = bitcast"), - "{label} fixture reached RS4GC before construction-time folds converged:\n{rewritten}" - ); - let live_bundle = rewritten - .lines() - .find(|line| line.contains("\"gc-live\"")) - .unwrap_or_else(|| panic!("{label} fixture lost every dynamic root:\n{rewritten}")); - assert!( - live_bundle.contains("%dp0"), - "{label} fixture lost every dynamic root:\n{rewritten}" - ); - assert!( - rewritten.contains("gc.relocate"), - "{label} fixture did not relocate a dynamic root:\n{rewritten}" - ); - } - - let emit = |ir: &str, name: &str| { - let context = Context::create(); - let module = parse_ir_text(&context, ir, name).expect("fixture parses"); - optimize_and_emit_module(&module, &target, &["-O3".into(), "-S".into()], true) - .expect("fixture emits assembly") - }; - // Both arms must be emitted under the SAME module name. The name - // becomes the module id, and on ELF the assembler writes it into the - // object as a `.file` directive — so two differently-named arms differ - // by that one line no matter how perfectly the code itself converged. - // Mach-O records no such directive, which is why naming them apart only - // ever failed on Linux (#8087). - let text = emit(&text_ir, "constant_fold_order"); - let folded = emit(&folded_ir, "constant_fold_order"); - assert_eq!( - text, folded, - "construction-time constant folding must converge before RS4GC assigns root liveness" - ); - - const PRE_FIX_PASSES: &str = "function(mem2reg),rewrite-statepoints-for-gc"; - let pre_fix_emit = |ir: &str, name: &str| { - let rewritten = statepoint_rewritten_ir_with_passes( - ir, - &target, - &format!("{name}_rewrite"), - PRE_FIX_PASSES, - ) - .expect("pre-fix pipeline rewrites fixture"); - let context = Context::create(); - let module = - parse_ir_text(&context, &rewritten, name).expect("rewritten fixture parses"); - let _shadow = crate::codegen::helpers::NativeRootsPin::shadow(); - ( - rewritten, - optimize_and_emit_module(&module, &target, &["-O3".into(), "-S".into()], false) - .expect("rewritten fixture emits assembly"), - ) - }; - let (pre_fix_text_ir, pre_fix_text) = pre_fix_emit(&text_ir, "pre_fix_text"); - let (_, pre_fix_folded) = pre_fix_emit(&folded_ir, "pre_fix_native"); - assert!( - pre_fix_text_ir - .lines() - .find(|line| line.contains("\"gc-live\"")) - .is_some_and(|line| line.contains("%cp0")), - "negative control must keep a constant-derived text root live across the safepoint:\n{pre_fix_text_ir}" - ); - assert_ne!( - pre_fix_text, pre_fix_folded, - "fixture must fail byte equality under the pre-#8065 pass order" - ); - } - - #[test] - fn rs4gc_honors_alwaysinline_before_rewriting_calls() { - let target = crate::codegen::default_target_triple(); - let ir = r#" -declare ptr addrspace(1) @alloc() - -define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" { -entry: - %unused = call ptr addrspace(1) @alloc() - ret ptr addrspace(1) %p -} - -define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" { -entry: - %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p) - ret ptr addrspace(1) %result -} -"#; - - const PRE_FIX_PASSES: &str = "function(mem2reg,sccp),rewrite-statepoints-for-gc"; - let before = - statepoint_rewritten_ir_with_passes(ir, &target, "alwaysinline_before", PRE_FIX_PASSES) - .expect("negative control rewrites the fixture"); - assert!( - before.lines().any(|line| { - line.contains("@llvm.experimental.gc.statepoint") && line.contains("@leaf") - }), - "negative control must leave the alwaysinline call as a statepoint:\n{before}" - ); - - let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_after") - .expect("shipped pipeline rewrites the inlined fixture"); - assert!( - !after.contains("@leaf"), - "alwaysinline callee and call must disappear before RS4GC:\n{after}" - ); - let live_bundle = after - .lines() - .find(|line| line.contains("@llvm.experimental.gc.statepoint")) - .unwrap_or_else(|| panic!("inlined allocation must remain a statepoint:\n{after}")); - assert!( - live_bundle.contains("\"gc-live\"") && live_bundle.contains("%p"), - "caller root must stay live through the inlined allocation:\n{after}" - ); - let relocation_results = relocation_results(&after); - let returned_pointers = returned_gc_pointers(&after); - assert_eq!( - returned_pointers.len(), - 1, - "fixture must retain exactly one return edge after inlining:\n{after}" - ); - assert!( - relocation_results.contains(returned_pointers[0]), - "caller must return the gc.relocate result, not the pre-statepoint root:\n{after}" - ); - } - - #[test] - fn rs4gc_rewrites_inlined_invoke_and_preserves_exception_edge() { - let target = crate::codegen::default_target_triple(); - let ir = r#" -declare ptr addrspace(1) @alloc() -declare i32 @perry_eh_personality(...) - -define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" personality ptr @perry_eh_personality { -entry: - %unused = invoke ptr addrspace(1) @alloc() - to label %ok unwind label %exception -ok: - ret ptr addrspace(1) %p -exception: - %landing = landingpad token cleanup - ret ptr addrspace(1) %p -} - -define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" personality ptr @perry_eh_personality { -entry: - %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p) - ret ptr addrspace(1) %result -} -"#; - - let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_invoke") - .expect("shipped pipeline rewrites an invoke in an inlined callee"); - assert!( - !after.contains("@leaf"), - "alwaysinline invoke callee must disappear before RS4GC:\n{after}" - ); - assert!( - after.lines().any(|line| { - line.contains("invoke token") && line.contains("@llvm.experimental.gc.statepoint") - }), - "inlined invoke must become a statepoint while retaining its unwind edge:\n{after}" - ); - assert!( - after.contains("landingpad token") - && after.lines().any(|line| line.trim() == "cleanup"), - "statepoint invoke must retain a verifier-valid exceptional pad:\n{after}" - ); - let relocation_results = relocation_results(&after); - let returned_pointers = returned_gc_pointers(&after); - assert_eq!( - returned_pointers.len(), - 1, - "inlined invoke fixture must retain one merged return edge:\n{after}" - ); - assert_eq!( - relocation_results.len(), - 2, - "normal and exceptional continuations must each relocate the root:\n{after}" - ); - let return_phi = after - .lines() - .find(|line| { - line.trim().starts_with(returned_pointers[0]) - && line.contains(" = phi ptr addrspace(1) ") - }) - .unwrap_or_else(|| { - panic!("invoke continuations must merge through the returned phi:\n{after}") - }); - assert!( - relocation_results - .iter() - .all(|relocated| return_phi.contains(*relocated)), - "returned phi must merge both gc.relocate results, not the pre-statepoint root:\n{after}" - ); - } - - /// Layer-2 readiness (#7174, engine-plan layer 0 -> 2): the in-process - /// pipeline can schedule `RewriteStatepointsForGC` at the pinned LLVM — - /// no `opt` subprocess, no version-skewed toolchain. This is the exact - /// mechanism #7108 measured as viable-but-blocked under text-plus-clang. - /// A statepoint lands at the may-GC call and the live GC pointer is - /// relocated across it — the property that makes the register-held- - /// pointer bug class unrepresentable. - #[test] - fn rs4gc_schedules_in_process() { - let context = Context::create(); - let ir = r#" -declare ptr addrspace(1) @alloc() - -define ptr addrspace(1) @f(ptr addrspace(1) %p) gc "statepoint-example" { -entry: - %q = call ptr addrspace(1) @alloc() - ret ptr addrspace(1) %p -} -"#; - let module = parse_ir_text(&context, ir, "rs4gc_probe").expect("probe parses"); - global_init(&[]); - let triple = TargetMachine::get_default_triple(); - let target = Target::from_triple(&triple).expect("host target"); - let tm = target - .create_target_machine( - &triple, - "", - "", - OptimizationLevel::None, - RelocMode::PIC, - CodeModel::Default, - ) - .expect("target machine"); - module - .run_passes( - "rewrite-statepoints-for-gc", - &tm, - PassBuilderOptions::create(), - ) - .expect("RS4GC pipeline runs in-process"); - let printed = module.print_to_string().to_string(); - assert!( - printed.contains("gc.statepoint"), - "no statepoint emitted:\n{printed}" - ); - assert!( - printed.contains("gc.relocate"), - "live GC pointer not relocated across the call:\n{printed}" - ); - module.verify().expect("statepoint IR verifies"); - } - - /// The initialized backend set must cover every triple the compile driver - /// can produce. `initialize_all()` cost +86.9 MB of static link for ~18 - /// unreachable backends; this pins the replacement, so narrowing it - /// further — or adding a target without initializing its backend — fails - /// here rather than at a user's compile. - #[test] - fn every_supported_triple_resolves_to_an_initialized_backend() { - global_init(&[]); - for triple in [ - "arm64-apple-macosx", - "aarch64-apple-ios", - "aarch64-apple-watchos", - "arm64_32-apple-watchos", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "aarch64-linux-android", - "x86_64-apple-darwin", - "x86_64-unknown-linux-gnu", - "x86_64-pc-windows-msvc", - "i686-unknown-linux-gnu", - ] { - let t = TargetTriple::create(triple); - assert!( - Target::from_triple(&t).is_ok(), - "{triple} has no initialized LLVM backend — the compile driver \ - can emit this triple, so `global_init` must initialize it" - ); - } - } - - /// #7327 CI regression: an empty CPU string makes LLVM pick `generic`, - /// which on aarch64 is ARMv8.0 and has no FEAT_JSCVT — so the - /// `llvm.aarch64.fjcvtzs` that codegen emits for any Apple arm64 triple - /// cannot be selected and the compile aborts. Clang defaults that triple to - /// `apple-m1`, which is the assumption `set_jscvt_for_target` already makes. - /// Reproduced with `PERRY_TARGET_CPU=generic`, which is the path CI took. - #[test] - fn apple_aarch64_defaults_to_a_cpu_with_feat_jscvt() { - for triple in [ - "arm64-apple-macosx", - "arm64-apple-darwin", - "aarch64-apple-darwin", - "arm64-apple-ios", - ] { - assert_eq!( - default_cpu_for_triple(triple), - "apple-m1", - "{triple} must not fall back to LLVM's ARMv8.0 `generic`: codegen \ - emits llvm.aarch64.fjcvtzs for Apple arm64 triples" - ); - } - // Everything else keeps LLVM's portable baseline, matching the clang - // path when no tuning flag is passed. - for triple in [ - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - ] { - assert_eq!(default_cpu_for_triple(triple), "", "{triple}"); - } - } - - /// `-S` used to be swallowed by the catch-all that ignores `-c`, so the - /// statepoint backends asked for assembly and were handed an object. The - /// failure was invisible here and surfaced two steps later as - /// `ld: unknown file type`, because #7314's compact-map rewriter rewrites - /// `.llvm_stackmaps` in assembly *text* and had nothing to rewrite. - #[test] - fn dash_s_requests_assembly_and_dash_c_does_not() { - let (_, _, _, _, emit_asm) = - interpret_plan_args(&["-O2".into(), "-S".into()]).expect("args parse"); - assert!(emit_asm, "-S must request assembly"); - - let (_, _, _, _, emit_asm) = - interpret_plan_args(&["-O2".into(), "-c".into()]).expect("args parse"); - assert!(!emit_asm, "-c must still request an object"); - } - - /// The property the wiring depends on: the same module emitted with - /// `FileType::Assembly` is assembler text carrying a stack-map section, - /// not an object. If this ever silently produced an object again, the - /// compact-map rewrite would find no `.llvm_stackmaps` to shrink and the - /// GC would be reading an empty map — the #7332 shape, a binary that - /// looks correct until a collection frees something live. - #[test] - fn assembly_emission_is_text_not_an_object() { - let context = Context::create(); - let ir = r#" -define i32 @f(i32 %x) { -entry: - %y = add i32 %x, 1 - ret i32 %y -} -"#; - let module = parse_ir_text(&context, ir, "asm_probe").expect("probe parses"); - global_init(&[]); - let triple = TargetMachine::get_default_triple(); - let target = Target::from_triple(&triple).expect("host target"); - let tm = target - .create_target_machine( - &triple, - "", - "", - OptimizationLevel::None, - RelocMode::PIC, - CodeModel::Default, - ) - .expect("target machine"); - - let asm = tm - .write_to_memory_buffer(&module, FileType::Assembly) - .expect("assembly emission"); - let text = String::from_utf8_lossy(asm.as_slice()).to_string(); - assert!( - text.contains(".globl") || text.contains(".global"), - "expected assembler directives, got:\n{}", - &text[..text.len().min(200)] - ); - - let obj = tm - .write_to_memory_buffer(&module, FileType::Object) - .expect("object emission"); - assert_ne!( - asm.as_slice(), - obj.as_slice(), - "assembly and object emission returned identical bytes — `-S` is \ - being ignored somewhere in the emission path" - ); - } - #[test] - fn tre_walk_budget_spellings() { - assert_eq!( - parse_tre_walk_budget(None), - TreWalkBudget::Cap(DEFAULT_TRE_MAX_ALLOCA_WALK) - ); - assert_eq!( - parse_tre_walk_budget(Some("")), - TreWalkBudget::Cap(DEFAULT_TRE_MAX_ALLOCA_WALK) - ); - assert_eq!(parse_tre_walk_budget(Some("0")), TreWalkBudget::Off); - assert_eq!(parse_tre_walk_budget(Some("off")), TreWalkBudget::Off); - assert_eq!(parse_tre_walk_budget(Some("false")), TreWalkBudget::Off); - assert_eq!( - parse_tre_walk_budget(Some(" 250000 ")), - TreWalkBudget::Cap(250_000) - ); - assert_eq!( - parse_tre_walk_budget(Some("lots")), - TreWalkBudget::Cap(DEFAULT_TRE_MAX_ALLOCA_WALK) - ); - } - - /// Two functions: `wide` has 4 allocas across 9 instructions (estimate - /// 36), `narrow` has one across 3 (estimate 3), and `decl` has no body. - fn alloca_walk_fixture() -> &'static str { - r#" -declare void @sink(ptr) - -define void @wide() { -entry: - %a = alloca i64 - %b = alloca i64 - %c = alloca i64 - %d = alloca i64 - call void @sink(ptr %a) - call void @sink(ptr %b) - call void @sink(ptr %c) - call void @sink(ptr %d) - ret void -} - -define void @narrow() { -entry: - %a = alloca i64 - call void @sink(ptr %a) - ret void -} -"# - } - - fn has_disable_tail_calls(module: &inkwell::module::Module<'_>, name: &str) -> bool { - module - .get_function(name) - .expect("fixture function exists") - .get_string_attribute( - inkwell::attributes::AttributeLoc::Function, - DISABLE_TAIL_CALLS_ATTR, - ) - .is_some_and(|attr| attr.get_string_value().to_bytes() == b"true") - } - - /// The budget is `allocas × instructions`, applied per function: only - /// the function over it is stamped, the boundary is exclusive, and - /// `off` stamps nothing. - #[test] - fn tre_budget_stamps_only_the_function_over_it() { - let context = Context::create(); - let module = parse_ir_text(&context, alloca_walk_fixture(), "tre_budget_fixture") - .expect("fixture parses"); - let wide = module.get_function("wide").expect("wide"); - let narrow = module.get_function("narrow").expect("narrow"); - assert_eq!(alloca_walk_factors(wide), (4, 9)); - assert_eq!(alloca_walk_factors(narrow), (1, 3)); - - assert!( - disable_tail_call_elim_over_budget(&module, TreWalkBudget::Off).is_empty(), - "a disabled budget stamps nothing" - ); - assert!(!has_disable_tail_calls(&module, "wide")); - - let exact = disable_tail_call_elim_over_budget(&module, TreWalkBudget::Cap(36)); - assert!(exact.is_empty(), "the cap is inclusive: {exact:?}"); - - let over = disable_tail_call_elim_over_budget(&module, TreWalkBudget::Cap(35)); - assert_eq!( - over, - vec![TreWalkOverBudget { - name: "wide".to_string(), - allocas: 4, - instructions: 9, - cap: 35, - }] - ); - assert!(has_disable_tail_calls(&module, "wide")); - assert!(!has_disable_tail_calls(&module, "narrow")); - let message = over[0].to_string(); - for needle in [ - "`wide`", - "4 allocas", - "9 instructions", - "estimate 36", - "budget 35", - "PERRY_LL_TRE_MAX_ALLOCA_WALK", - "#8883", - ] { - assert!( - message.contains(needle), - "{needle} missing from:\n{message}" - ); - } - assert!( - !message.contains("optnone"), - "the budget must never read as a demotion:\n{message}" - ); - } - - /// A self-recursive tail call that TailCallElim turns into a loop at - /// the pinned LLVM: with no attribute the recursive `call` disappears, - /// with `"disable-tail-calls"="true"` (exactly what the budget stamps) - /// it survives the full `default` pipeline — so the lever the - /// budget pulls is live, not merely spelled. - fn tail_recursive_fixture(attrs: &str) -> String { - format!( - "define i64 @count_down(i64 %n, i64 %acc) noinline {attrs} {{\n\ - entry:\n\ - \x20 %done = icmp eq i64 %n, 0\n\ - \x20 br i1 %done, label %ret, label %rec\n\ - rec:\n\ - \x20 %n1 = sub i64 %n, 1\n\ - \x20 %acc1 = add i64 %acc, %n\n\ - \x20 %r = call i64 @count_down(i64 %n1, i64 %acc1)\n\ - \x20 ret i64 %r\n\ - ret:\n\ - \x20 ret i64 %acc\n\ - }}\n" - ) - } - - #[test] - fn disable_tail_calls_attribute_stops_tail_call_elim_at_the_pinned_llvm() { - let target = crate::codegen::default_target_triple(); - let with_tre = statepoint_rewritten_ir_with_passes( - &tail_recursive_fixture(""), - &target, - "tre_control", - "default", - ) - .expect("control optimizes"); - assert!( - !with_tre.contains("call i64 @count_down"), - "control: TailCallElim must turn the tail recursion into a loop, or this test \ - cannot tell the attribute apart from a no-op:\n{with_tre}" - ); - - let without_tre = statepoint_rewritten_ir_with_passes( - &tail_recursive_fixture(&format!("\"{DISABLE_TAIL_CALLS_ATTR}\"=\"true\"")), - &target, - "tre_disabled", - "default", - ) - .expect("attributed fixture optimizes"); - assert!( - without_tre.contains("call i64 @count_down"), - "the attribute must keep TailCallElim off the function:\n{without_tre}" - ); - } - - /// The budget is wired into the shipped emission path: under a cap of - /// zero every function with an alloca is stamped before `default` - /// runs, the per-unit stats name it, and the unit still emits. - #[test] - fn tre_budget_is_applied_by_the_shipped_pipeline() { - global_init(&[]); - let target = crate::codegen::default_target_triple(); - let context = Context::create(); - let module = parse_ir_text(&context, alloca_walk_fixture(), "tre_budget_shipped") - .expect("fixture parses"); - let mut stats = UnitCodegenStats::default(); - let object = with_test_tre_walk_budget(0, || { - optimize_and_emit_module_with_stats( - &module, - &target, - &["-Os".into(), "-c".into()], - false, - Some(&mut stats), - ) - }) - .expect("a stamped module still optimizes and emits"); - assert!(!object.is_empty()); - let mut names: Vec<&str> = stats - .tail_call_elim_skipped - .iter() - .map(|over| over.name.as_str()) - .collect(); - names.sort_unstable(); - assert_eq!(names, ["narrow", "wide"]); - - // -O0 runs no TailCallElim, so nothing is stamped there. - let module = parse_ir_text(&context, alloca_walk_fixture(), "tre_budget_o0") - .expect("fixture parses"); - let mut stats = UnitCodegenStats::default(); - with_test_tre_walk_budget(0, || { - optimize_and_emit_module_with_stats( - &module, - &target, - &["-O0".into(), "-c".into()], - false, - Some(&mut stats), - ) - }) - .expect("-O0 emits"); - assert!(stats.tail_call_elim_skipped.is_empty()); - assert!(!has_disable_tail_calls(&module, "wide")); - } -} diff --git a/crates/perry-codegen/src/inprocess/optimize_emit.rs b/crates/perry-codegen/src/inprocess/optimize_emit.rs new file mode 100644 index 0000000000..a0f1e240ec --- /dev/null +++ b/crates/perry-codegen/src/inprocess/optimize_emit.rs @@ -0,0 +1,1316 @@ +//! LLVM optimize-and-emit for the in-process backend. +//! +//! Split out of `inprocess.rs` to keep that file under the 2000-line size gate. + +use super::*; + +pub(super) fn optimize_and_emit( + module: &inkwell::module::Module<'_>, + effective_target: &str, + opt: char, + mcpu_native: bool, + explicit_cpu: Option<&str>, + mllvm: &[String], + emit_asm: bool, + native_roots: bool, + mut stats: Option<&mut UnitCodegenStats>, +) -> Result> { + global_init(mllvm); + announce(); + + module + .verify() + .map_err(|e| anyhow!("LLVM verifier rejected module:\n{}", e.to_string()))?; + + let triple = TargetTriple::create(effective_target); + let target = Target::from_triple(&triple) + .map_err(|e| anyhow!("no LLVM target for `{effective_target}`: {e}"))?; + let (cpu, features) = if mcpu_native { + ( + TargetMachine::get_host_cpu_name() + .to_string_lossy() + .into_owned(), + TargetMachine::get_host_cpu_features() + .to_string_lossy() + .into_owned(), + ) + } else if let Some(cpu) = explicit_cpu { + (cpu.to_string(), String::new()) + } else { + ( + default_cpu_for_triple(effective_target).to_string(), + String::new(), + ) + }; + let opt_level = match opt { + '0' => OptimizationLevel::None, + '1' => OptimizationLevel::Less, + '2' | 's' | 'z' => OptimizationLevel::Default, + _ => OptimizationLevel::Aggressive, + }; + let tm = target + .create_target_machine( + &triple, + &cpu, + &features, + opt_level, + RelocMode::PIC, + CodeModel::Default, + ) + .ok_or_else(|| anyhow!("failed to create TargetMachine for `{effective_target}`"))?; + + // Same trust order as the subprocess path: `-target` wins over whatever + // triple the module text states, and the module optimizes under the + // machine's real datalayout. + module.set_triple(&triple); + module.set_data_layout(&tm.get_target_data().get_data_layout()); + + // RS4GC must run BEFORE the optimization pipeline, and — critically — in + // this process, against this LLVM. + // + // The external path shells `rewrite-statepoints-for-gc` out to an `opt` + // binary and then hands the rewritten IR to `clang -c`. When those are + // different LLVM versions (Homebrew 22 and Apple clang 21 is the ordinary + // macOS case) the emitted IR uses constructs the older parser rejects, and + // the compile dies with `error: unterminated attribute group`. That is why + // RS4GC needed `PERRY_LLVM_CLANG` pointed at a version-matched toolchain, + // and why it did not work on a stock install at all. + // + // Here the same `TargetMachine` runs the pass and emits the object, so the + // skew cannot exist. This matters beyond convenience: RS4GC is the only + // backend that can root an `invoke`, and since #7302 every call inside a + // `try` is one — 26% of the gap suite (128 of 479 files) contains a `try`, + // which the explicit bridge refuses outright (#7327/#7330). + if native_roots { + // Sizes before the rewrite: the budget message below names them, and + // the per-unit report compares them with the post-rewrite census. + let budget = rs4gc_instruction_budget(); + let preflight_cap = crate::codegen::helpers::root_spill_relocation_threshold(); + let rewritten_functions = rs4gc_functions(module); + let pre_sizes = if budget == RewriteBudget::Off && preflight_cap == 0 && stats.is_none() { + std::collections::HashMap::new() + } else { + pre_rewrite_sizes(module) + }; + if let Some(stats) = stats.as_deref_mut() { + stats.functions = pre_sizes.len(); + stats.pre_rewrite_instructions = pre_sizes.values().sum(); + stats.pre_rewrite_widest = pre_sizes + .iter() + .max_by_key(|(_, n)| **n) + .map(|(name, n)| (name.clone(), *n)); + } + // The source-level estimate is intentionally cheap but can miss + // codegen expansion (one expression becoming many collecting helper + // calls). Check the actual constructed CallBase/root shape before + // asking RS4GC to perform the potentially super-linear rewrite. + enforce_rs4gc_preflight_budget(module, preflight_cap, &pre_sizes, &rewritten_functions)?; + let rewrite_started = std::time::Instant::now(); + module + .run_passes(STATEPOINT_REWRITE_PASSES, &tm, PassBuilderOptions::create()) + .map_err(|e| { + anyhow!( + "in-process rewrite-statepoints-for-gc failed:\n{}", + e.to_string() + ) + })?; + // Verify the rewritten module before it reaches the backend. RS4GC + // has produced verifier-invalid IR in the wild (#8121: it wrapped an + // inline-asm barrier into a gc.statepoint), and unlike the external + // `opt` path — whose verifier aborts with the broken instruction — + // the in-process pipeline would feed the broken module straight to + // ISel, where it dies as a bare SIGBUS with no diagnostic. + module.verify().map_err(|e| { + anyhow!( + "in-process rewrite-statepoints-for-gc produced a module the \ + verifier rejects (this is a Perry codegen bug — the input \ + shape must be exempted or fixed):\n{}", + e.to_string() + ) + })?; + if let Some(stats) = stats.as_deref_mut() { + stats.rewrite_secs = rewrite_started.elapsed().as_secs_f64(); + let (_, total, widest) = module_instruction_census(module); + stats.post_rewrite_instructions = total; + stats.post_rewrite_widest = widest; + } + // The relocation-fan-out backstop (#8583/#8679): stop before the + // super-linear optimizer and ask codegen to retry the named functions + // with precise shadow-frame roots. The retry keeps this same pipeline + // and optimization level; only the GC-root representation changes. + enforce_rs4gc_instruction_budget(module, budget, &pre_sizes, &rewritten_functions)?; + } + + let pipeline = match opt { + '0' => "default", + '1' => "default", + '2' => "default", + 's' => "default", + 'z' => "default", + _ => "default", + }; + // TailCallElim runs inside every `default` function-simplification + // pipeline; bound its alloca walk on the module the pipeline will see + // (#8883). `-O0` runs no TRE, so there is nothing to bound. + if opt != '0' { + let skipped = disable_tail_call_elim_over_budget(module, tre_walk_budget()); + for over in &skipped { + eprintln!("perry: {over}"); + } + if let Some(stats) = stats.as_deref_mut() { + stats.tail_call_elim_skipped = skipped; + } + } + let optimize_started = std::time::Instant::now(); + module + .run_passes(pipeline, &tm, PassBuilderOptions::create()) + .map_err(|e| anyhow!("pass pipeline `{pipeline}` failed:\n{}", e.to_string()))?; + if let Some(stats) = stats.as_deref_mut() { + stats.optimize_secs = optimize_started.elapsed().as_secs_f64(); + } + + // The IR pipeline above has already done the requested optimization. For + // an extreme generated function, LLVM's optimized *machine* pipeline can + // still become super-linear in instruction selection / LiveIntervals / + // register allocation. Use an O0 target machine only for final emission + // of that unit; ordinary units keep `tm`, and the optimized IR is not + // rebuilt or demoted. + let fast_emit = if opt == '0' { + None + } else { + fast_emit_fallback(module, fast_emit_budget()) + }; + if let Some(fallback) = &fast_emit { + eprintln!("perry: {fallback}"); + } + if let Some(stats) = stats.as_deref_mut() { + stats.fast_emit_fallback = fast_emit.clone(); + } + let fast_tm = if fast_emit.is_some() { + Some( + target + .create_target_machine( + &triple, + &cpu, + &features, + OptimizationLevel::None, + RelocMode::PIC, + CodeModel::Default, + ) + .ok_or_else(|| { + anyhow!( + "failed to create bounded O0 emission TargetMachine for \ + `{effective_target}`" + ) + })?, + ) + } else { + None + }; + let emit_tm = fast_tm.as_ref().unwrap_or(&tm); + + let kind = if emit_asm { + FileType::Assembly + } else { + FileType::Object + }; + let emit_started = std::time::Instant::now(); + let obj = emit_tm + .write_to_memory_buffer(module, kind) + .map_err(|e| anyhow!("{kind:?} emission failed:\n{}", e.to_string()))?; + if let Some(stats) = stats { + stats.emit_secs = emit_started.elapsed().as_secs_f64(); + } + Ok(obj.as_slice().to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn relocation_results(ir: &str) -> std::collections::HashSet<&str> { + ir.lines() + .filter(|line| line.contains("@llvm.experimental.gc.relocate")) + .filter_map(|line| line.trim().split_once(" = ").map(|(result, _)| result)) + .collect() + } + + fn returned_gc_pointers(ir: &str) -> Vec<&str> { + ir.lines() + .filter_map(|line| { + line.trim() + .strip_prefix("ret ptr addrspace(1) ") + .and_then(|value| value.split_whitespace().next()) + }) + .collect() + } + + fn asm_barrier_fixture(leaf_attr: &str) -> String { + format!( + "declare i64 @may_collect()\n\n\ + define i64 @f(i64 %a) gc \"statepoint-example\" {{\n\ + entry:\n\ + \x20 %slot = alloca ptr addrspace(1)\n\ + \x20 %p = inttoptr i64 %a to ptr addrspace(1)\n\ + \x20 store ptr addrspace(1) %p, ptr %slot\n\ + \x20 call void asm sideeffect \"\", \"\"(){leaf_attr}\n\ + \x20 %t = call i64 @may_collect()\n\ + \x20 %after = load ptr addrspace(1), ptr %slot\n\ + \x20 %bits = ptrtoint ptr addrspace(1) %after to i64\n\ + \x20 %r = add i64 %t, %bits\n\ + \x20 ret i64 %r\n\ + }}\n" + ) + } + + #[test] + fn gc_leaf_asm_barrier_survives_rs4gc_unwrapped() { + // The shipped emitters stamp the loop-preservation barrier + // `"gc-leaf-function"`; RS4GC must leave it as a plain inline-asm + // call while still statepointing the real call next to it. + let rewritten = statepoint_rewritten_ir( + &asm_barrier_fixture(" \"gc-leaf-function\""), + "arm64-apple-darwin", + "asm_barrier_leaf", + ) + .expect("attributed barrier must survive the rewrite"); + assert!( + rewritten.contains("call void asm sideeffect"), + "barrier must remain a plain inline-asm call:\n{rewritten}" + ); + assert!( + !rewritten.contains("elementtype(void ()) asm"), + "barrier must not be statepoint-wrapped:\n{rewritten}" + ); + assert!( + rewritten.contains("@llvm.experimental.gc.statepoint"), + "the genuine call must still be statepointed:\n{rewritten}" + ); + } + + #[test] + fn unattributed_asm_barrier_is_rejected_not_miscompiled() { + // Sabotage arm: without the attribute RS4GC wraps the asm into a + // gc.statepoint whose callee is inline asm — invalid IR. The + // pipeline must fail verification loudly (#8121's SIGBUS shape), + // proving the leaf test above can actually fail. + let result = statepoint_rewritten_ir( + &asm_barrier_fixture(""), + "arm64-apple-darwin", + "asm_barrier_broken", + ); + assert!( + result.is_err(), + "an unattributed barrier must be rejected by the verifier" + ); + } + + #[test] + fn rewrite_budget_spellings() { + assert_eq!( + parse_rewrite_budget(None), + RewriteBudget::Error(DEFAULT_RS4GC_MAX_INSTRS) + ); + assert_eq!(parse_rewrite_budget(Some("0")), RewriteBudget::Off); + assert_eq!(parse_rewrite_budget(Some("off")), RewriteBudget::Off); + assert_eq!( + parse_rewrite_budget(Some(" 250000 ")), + RewriteBudget::Error(250_000) + ); + assert_eq!( + parse_rewrite_budget(Some("warn:4096")), + RewriteBudget::Warn(4096) + ); + assert_eq!(parse_rewrite_budget(Some("warn:0")), RewriteBudget::Off); + // Unparsable values keep the default rather than silently disabling. + assert_eq!( + parse_rewrite_budget(Some("lots")), + RewriteBudget::Error(DEFAULT_RS4GC_MAX_INSTRS) + ); + } + + /// The source-level estimate is only a fast first line of defence. This + /// fixture pins the constructed-IR backstop: managed-root allocas count, + /// ordinary calls count, explicit GC-leaf calls and LLVM intrinsics do + /// not, and only functions which will actually enter RS4GC are governed. + #[test] + fn rs4gc_preflight_uses_constructed_roots_and_non_leaf_calls() { + let fixture = r#" +declare i64 @may_collect() +declare i64 @leaf() +declare void @llvm.donothing() + +define i64 @hot() gc "statepoint-example" { +entry: + %root = alloca ptr addrspace(1) + %plain = alloca i64 + %a = call i64 @may_collect() + %b = call i64 @may_collect() + %c = call i64 @leaf() "gc-leaf-function" + call void @llvm.donothing() + %p = load ptr addrspace(1), ptr %root + %bits = ptrtoint ptr addrspace(1) %p to i64 + %sum = add i64 %a, %b + %sum2 = add i64 %sum, %c + %out = add i64 %sum2, %bits + ret i64 %out +} + +define i64 @shadow() { +entry: + %root = alloca ptr addrspace(1) + %a = call i64 @may_collect() + ret i64 %a +} +"#; + let context = Context::create(); + let module = parse_ir_text(&context, fixture, "preflight_fixture").expect("fixture parses"); + let hot = module.get_function("hot").expect("hot"); + assert_eq!( + rs4gc_preflight_factors(hot), + (1, 2), + "plain allocas, leaf calls and intrinsics do not add RS4GC work" + ); + + // (one constructed root + two possible call-result roots) x two + // safepoints = six estimated relocations. The boundary is exclusive. + let rewritten_functions = rs4gc_functions(&module); + assert_eq!( + rs4gc_preflight_violations(&module, 5, &rewritten_functions), + vec![("hot".to_string(), 1, 2, 6)] + ); + assert!(rs4gc_preflight_violations(&module, 6, &rewritten_functions).is_empty()); + assert!(rs4gc_preflight_violations(&module, 0, &rewritten_functions).is_empty()); + + let pre = pre_rewrite_sizes(&module); + let err = enforce_rs4gc_preflight_budget(&module, 5, &pre, &rewritten_functions) + .expect_err("the constructed shape requests a spill retry"); + let retry = rs4gc_budget_retry(&err).expect("the request stays typed"); + assert_eq!(retry.len(), 1); + assert_eq!(retry[0].name, "hot"); + assert_eq!(retry[0].pre_instructions, pre.get("hot").copied()); + assert_eq!( + retry[0].cause, + Rs4gcBudgetCause::PreRewrite { + root_allocas: 1, + safepoints: 2, + estimated_relocations: 6, + } + ); + assert_eq!(retry[0].cap, 5); + let msg = format!("{err:#}"); + for needle in [ + "before rewrite-statepoints-for-gc", + "`hot`", + "1 managed-root allocas", + "2 non-leaf call sites", + "predicts 6 relocations", + "budget 5", + "PERRY_ROOT_SPILL_RELOCATIONS", + "re-lower", + ] { + assert!( + msg.contains(needle), + "message must carry {needle:?}:\n{msg}" + ); + } + + let no_rewritten_functions = std::collections::HashSet::new(); + enforce_rs4gc_preflight_budget(&module, 1, &pre, &no_rewritten_functions) + .expect("a shadow-root function is outside the preflight budget"); + } + + /// Six gc values live across forty safepoints: ~60 instructions before + /// `rewrite-statepoints-for-gc`, a few hundred after (each statepoint + /// relocates every live value). A budget between the two is exceeded + /// only by the post-rewrite module — which is the property the + /// assertion exists for. Counting BEFORE the rewrite (the #8421 + /// replacement knob's mistake) would make `after` empty and fail here. + fn relocation_fanout_fixture() -> String { + let mut ir = String::from( + "declare i64 @may_collect()\n\n\ + define i64 @f(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5) gc \"statepoint-example\" {\n\ + entry:\n", + ); + for i in 0..6 { + ir.push_str(&format!( + " %p{i} = inttoptr i64 %a{i} to ptr addrspace(1)\n" + )); + } + for c in 0..40 { + ir.push_str(&format!(" %c{c} = call i64 @may_collect()\n")); + } + for i in 0..6 { + ir.push_str(&format!( + " %b{i} = ptrtoint ptr addrspace(1) %p{i} to i64\n" + )); + } + ir.push_str( + " %s0 = add i64 %b0, %b1\n %s1 = add i64 %s0, %b2\n %s2 = add i64 %s1, %b3\n\ + \x20 %s3 = add i64 %s2, %b4\n %s4 = add i64 %s3, %b5\n %s5 = add i64 %s4, %c0\n\ + \x20 %s6 = add i64 %s5, %c39\n ret i64 %s6\n}\n", + ); + ir + } + + #[test] + fn rs4gc_budget_fires_only_on_the_rewritten_module() { + global_init(&[]); + let target = "arm64-apple-darwin"; + let fixture = relocation_fanout_fixture(); + let rewritten = statepoint_rewritten_ir(&fixture, target, "fanout_budget") + .expect("fan-out fixture must run RS4GC"); + + let context = Context::create(); + let before = parse_ir_text(&context, &fixture, "fanout_before").expect("fixture parses"); + let after = parse_ir_text(&context, &rewritten, "fanout_after").expect("rewritten parses"); + let pre = pre_rewrite_sizes(&before); + let rewritten_functions = rs4gc_functions(&before); + let pre_f = pre["f"]; + let (_, post_total, post_widest) = module_instruction_census(&after); + let post_f = post_widest.as_ref().map(|(_, n)| *n).unwrap_or(0); + assert!( + post_f > 3 * pre_f, + "fixture must grow under relocation fan-out (pre {pre_f}, post {post_f}):\n{rewritten}" + ); + assert_eq!(post_total, post_f, "one defined function"); + let cap = pre_f + (post_f - pre_f) / 2; + + assert!( + rs4gc_budget_violations(&before, cap, &rewritten_functions).is_empty(), + "the pre-rewrite module is under the budget by construction" + ); + let over = rs4gc_budget_violations(&after, cap, &rewritten_functions); + assert_eq!( + over.len(), + 1, + "exactly the rewritten body is over: {over:?}" + ); + assert_eq!(over[0].0, "f"); + assert_eq!(over[0].1, post_f); + + let err = enforce_rs4gc_instruction_budget( + &after, + RewriteBudget::Error(cap), + &pre, + &rewritten_functions, + ) + .expect_err("the default spelling requests a spill retry"); + let retry = rs4gc_budget_retry(&err).expect("the request stays typed"); + assert_eq!(retry.len(), 1); + assert_eq!(retry[0].name, "f"); + assert_eq!(retry[0].pre_instructions, Some(pre_f)); + assert_eq!( + retry[0].cause, + Rs4gcBudgetCause::PostRewrite { + post_instructions: post_f + } + ); + assert_eq!(retry[0].cap, cap); + let msg = format!("{err:#}"); + for needle in [ + "`f`", + &format!("to {post_f} instructions"), + &format!("it was {pre_f} before"), + &format!("budget is {cap}"), + "PERRY_LL_RS4GC_MAX_INSTRS", + "re-lower", + "#8679", + ] { + assert!( + msg.contains(needle), + "message must carry {needle:?}:\n{msg}" + ); + } + assert!( + !msg.contains("optnone"), + "the budget is an assertion, never a demotion:\n{msg}" + ); + enforce_rs4gc_instruction_budget( + &after, + RewriteBudget::Warn(cap), + &pre, + &rewritten_functions, + ) + .expect("warn spelling does not retry"); + enforce_rs4gc_instruction_budget(&after, RewriteBudget::Off, &pre, &rewritten_functions) + .expect("off spelling does not retry"); + enforce_rs4gc_instruction_budget( + &after, + RewriteBudget::Error(post_f), + &pre, + &rewritten_functions, + ) + .expect("a budget at the exact size is not exceeded"); + + // The retry removes the function's GC strategy. Its ordinary shadow + // body may itself exceed a deliberately tiny test cap, but it must not + // request the same spill forever: only functions that entered RS4GC + // are governed by this relocation-fan-out budget. + let no_rewritten_functions = std::collections::HashSet::new(); + enforce_rs4gc_instruction_budget( + &after, + RewriteBudget::Error(cap), + &pre, + &no_rewritten_functions, + ) + .expect("a shadow-spilled function is outside the RS4GC budget"); + } + + fn constant_fold_order_fixture(folded: bool) -> String { + let mut ir = String::from( + "declare i64 @may_collect()\n\ndefine i64 @f(i64 %d0, i64 %d1, i64 %d2, i64 %d3, i64 %d4, i64 %d5, i64 %d6, i64 %d7) gc \"statepoint-example\" {\nentry:\n", + ); + for i in 0..8 { + ir.push_str(&format!(" %cslot{i} = alloca ptr addrspace(1)\n")); + if folded { + ir.push_str(&format!( + " store ptr addrspace(1) inttoptr (i64 9222246136947933185 to ptr addrspace(1)), ptr %cslot{i}\n" + )); + } else { + ir.push_str(&format!( + " %cb{i} = bitcast double 0x7FFC000000000001 to i64\n %cp{i} = inttoptr i64 %cb{i} to ptr addrspace(1)\n store ptr addrspace(1) %cp{i}, ptr %cslot{i}\n" + )); + } + } + for i in 0..8 { + ir.push_str(&format!( + " %dslot{i} = alloca ptr addrspace(1)\n %dp{i} = inttoptr i64 %d{i} to ptr addrspace(1)\n store ptr addrspace(1) %dp{i}, ptr %dslot{i}\n" + )); + } + ir.push_str(" %sp = call i64 @may_collect()\n"); + for i in 0..8 { + ir.push_str(&format!( + " %after{i} = load ptr addrspace(1), ptr %dslot{i}\n %bits{i} = ptrtoint ptr addrspace(1) %after{i} to i64\n" + )); + } + for i in 0..8 { + ir.push_str(&format!( + " %cafter{i} = load ptr addrspace(1), ptr %cslot{i}\n %cbits{i} = ptrtoint ptr addrspace(1) %cafter{i} to i64\n" + )); + } + ir.push_str(" %x1 = xor i64 %bits0, %bits1\n"); + for i in 2..8 { + ir.push_str(&format!(" %x{i} = xor i64 %x{}, %bits{i}\n", i - 1)); + } + ir.push_str(" %y0 = xor i64 %x7, %cbits0\n"); + for i in 1..8 { + ir.push_str(&format!(" %y{i} = xor i64 %y{}, %cbits{i}\n", i - 1)); + } + ir.push_str(" ret i64 %y7\n}\n"); + ir + } + + #[test] + fn rs4gc_canonicalizes_construction_time_folds_before_root_liveness() { + let _native = crate::codegen::helpers::NativeRootsPin::native(); + let target = crate::codegen::default_target_triple(); + let text_ir = constant_fold_order_fixture(false); + let folded_ir = constant_fold_order_fixture(true); + + for (label, ir) in [("text", &text_ir), ("folded", &folded_ir)] { + let rewritten = statepoint_rewritten_ir(ir, &target, label) + .unwrap_or_else(|e| panic!("{label} fixture must run RS4GC: {e:#}")); + assert!( + !rewritten.contains("%cb0 = bitcast"), + "{label} fixture reached RS4GC before construction-time folds converged:\n{rewritten}" + ); + let live_bundle = rewritten + .lines() + .find(|line| line.contains("\"gc-live\"")) + .unwrap_or_else(|| panic!("{label} fixture lost every dynamic root:\n{rewritten}")); + assert!( + live_bundle.contains("%dp0"), + "{label} fixture lost every dynamic root:\n{rewritten}" + ); + assert!( + rewritten.contains("gc.relocate"), + "{label} fixture did not relocate a dynamic root:\n{rewritten}" + ); + } + + let emit = |ir: &str, name: &str| { + let context = Context::create(); + let module = parse_ir_text(&context, ir, name).expect("fixture parses"); + optimize_and_emit_module(&module, &target, &["-O3".into(), "-S".into()], true) + .expect("fixture emits assembly") + }; + // Both arms must be emitted under the SAME module name. The name + // becomes the module id, and on ELF the assembler writes it into the + // object as a `.file` directive — so two differently-named arms differ + // by that one line no matter how perfectly the code itself converged. + // Mach-O records no such directive, which is why naming them apart only + // ever failed on Linux (#8087). + let text = emit(&text_ir, "constant_fold_order"); + let folded = emit(&folded_ir, "constant_fold_order"); + assert_eq!( + text, folded, + "construction-time constant folding must converge before RS4GC assigns root liveness" + ); + + const PRE_FIX_PASSES: &str = "function(mem2reg),rewrite-statepoints-for-gc"; + let pre_fix_emit = |ir: &str, name: &str| { + let rewritten = statepoint_rewritten_ir_with_passes( + ir, + &target, + &format!("{name}_rewrite"), + PRE_FIX_PASSES, + ) + .expect("pre-fix pipeline rewrites fixture"); + let context = Context::create(); + let module = + parse_ir_text(&context, &rewritten, name).expect("rewritten fixture parses"); + let _shadow = crate::codegen::helpers::NativeRootsPin::shadow(); + ( + rewritten, + optimize_and_emit_module(&module, &target, &["-O3".into(), "-S".into()], false) + .expect("rewritten fixture emits assembly"), + ) + }; + let (pre_fix_text_ir, pre_fix_text) = pre_fix_emit(&text_ir, "pre_fix_text"); + let (_, pre_fix_folded) = pre_fix_emit(&folded_ir, "pre_fix_native"); + assert!( + pre_fix_text_ir + .lines() + .find(|line| line.contains("\"gc-live\"")) + .is_some_and(|line| line.contains("%cp0")), + "negative control must keep a constant-derived text root live across the safepoint:\n{pre_fix_text_ir}" + ); + assert_ne!( + pre_fix_text, pre_fix_folded, + "fixture must fail byte equality under the pre-#8065 pass order" + ); + } + + #[test] + fn rs4gc_honors_alwaysinline_before_rewriting_calls() { + let target = crate::codegen::default_target_triple(); + let ir = r#" +declare ptr addrspace(1) @alloc() + +define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" { +entry: + %unused = call ptr addrspace(1) @alloc() + ret ptr addrspace(1) %p +} + +define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" { +entry: + %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p) + ret ptr addrspace(1) %result +} +"#; + + const PRE_FIX_PASSES: &str = "function(mem2reg,sccp),rewrite-statepoints-for-gc"; + let before = + statepoint_rewritten_ir_with_passes(ir, &target, "alwaysinline_before", PRE_FIX_PASSES) + .expect("negative control rewrites the fixture"); + assert!( + before.lines().any(|line| { + line.contains("@llvm.experimental.gc.statepoint") && line.contains("@leaf") + }), + "negative control must leave the alwaysinline call as a statepoint:\n{before}" + ); + + let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_after") + .expect("shipped pipeline rewrites the inlined fixture"); + assert!( + !after.contains("@leaf"), + "alwaysinline callee and call must disappear before RS4GC:\n{after}" + ); + let live_bundle = after + .lines() + .find(|line| line.contains("@llvm.experimental.gc.statepoint")) + .unwrap_or_else(|| panic!("inlined allocation must remain a statepoint:\n{after}")); + assert!( + live_bundle.contains("\"gc-live\"") && live_bundle.contains("%p"), + "caller root must stay live through the inlined allocation:\n{after}" + ); + let relocation_results = relocation_results(&after); + let returned_pointers = returned_gc_pointers(&after); + assert_eq!( + returned_pointers.len(), + 1, + "fixture must retain exactly one return edge after inlining:\n{after}" + ); + assert!( + relocation_results.contains(returned_pointers[0]), + "caller must return the gc.relocate result, not the pre-statepoint root:\n{after}" + ); + } + + #[test] + fn rs4gc_rewrites_inlined_invoke_and_preserves_exception_edge() { + let target = crate::codegen::default_target_triple(); + let ir = r#" +declare ptr addrspace(1) @alloc() +declare i32 @perry_eh_personality(...) + +define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" personality ptr @perry_eh_personality { +entry: + %unused = invoke ptr addrspace(1) @alloc() + to label %ok unwind label %exception +ok: + ret ptr addrspace(1) %p +exception: + %landing = landingpad token cleanup + ret ptr addrspace(1) %p +} + +define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" personality ptr @perry_eh_personality { +entry: + %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p) + ret ptr addrspace(1) %result +} +"#; + + let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_invoke") + .expect("shipped pipeline rewrites an invoke in an inlined callee"); + assert!( + !after.contains("@leaf"), + "alwaysinline invoke callee must disappear before RS4GC:\n{after}" + ); + assert!( + after.lines().any(|line| { + line.contains("invoke token") && line.contains("@llvm.experimental.gc.statepoint") + }), + "inlined invoke must become a statepoint while retaining its unwind edge:\n{after}" + ); + assert!( + after.contains("landingpad token") + && after.lines().any(|line| line.trim() == "cleanup"), + "statepoint invoke must retain a verifier-valid exceptional pad:\n{after}" + ); + let relocation_results = relocation_results(&after); + let returned_pointers = returned_gc_pointers(&after); + assert_eq!( + returned_pointers.len(), + 1, + "inlined invoke fixture must retain one merged return edge:\n{after}" + ); + assert_eq!( + relocation_results.len(), + 2, + "normal and exceptional continuations must each relocate the root:\n{after}" + ); + let return_phi = after + .lines() + .find(|line| { + line.trim().starts_with(returned_pointers[0]) + && line.contains(" = phi ptr addrspace(1) ") + }) + .unwrap_or_else(|| { + panic!("invoke continuations must merge through the returned phi:\n{after}") + }); + assert!( + relocation_results + .iter() + .all(|relocated| return_phi.contains(*relocated)), + "returned phi must merge both gc.relocate results, not the pre-statepoint root:\n{after}" + ); + } + + /// Layer-2 readiness (#7174, engine-plan layer 0 -> 2): the in-process + /// pipeline can schedule `RewriteStatepointsForGC` at the pinned LLVM — + /// no `opt` subprocess, no version-skewed toolchain. This is the exact + /// mechanism #7108 measured as viable-but-blocked under text-plus-clang. + /// A statepoint lands at the may-GC call and the live GC pointer is + /// relocated across it — the property that makes the register-held- + /// pointer bug class unrepresentable. + #[test] + fn rs4gc_schedules_in_process() { + let context = Context::create(); + let ir = r#" +declare ptr addrspace(1) @alloc() + +define ptr addrspace(1) @f(ptr addrspace(1) %p) gc "statepoint-example" { +entry: + %q = call ptr addrspace(1) @alloc() + ret ptr addrspace(1) %p +} +"#; + let module = parse_ir_text(&context, ir, "rs4gc_probe").expect("probe parses"); + global_init(&[]); + let triple = TargetMachine::get_default_triple(); + let target = Target::from_triple(&triple).expect("host target"); + let tm = target + .create_target_machine( + &triple, + "", + "", + OptimizationLevel::None, + RelocMode::PIC, + CodeModel::Default, + ) + .expect("target machine"); + module + .run_passes( + "rewrite-statepoints-for-gc", + &tm, + PassBuilderOptions::create(), + ) + .expect("RS4GC pipeline runs in-process"); + let printed = module.print_to_string().to_string(); + assert!( + printed.contains("gc.statepoint"), + "no statepoint emitted:\n{printed}" + ); + assert!( + printed.contains("gc.relocate"), + "live GC pointer not relocated across the call:\n{printed}" + ); + module.verify().expect("statepoint IR verifies"); + } + + /// The initialized backend set must cover every triple the compile driver + /// can produce. `initialize_all()` cost +86.9 MB of static link for ~18 + /// unreachable backends; this pins the replacement, so narrowing it + /// further — or adding a target without initializing its backend — fails + /// here rather than at a user's compile. + #[test] + fn every_supported_triple_resolves_to_an_initialized_backend() { + global_init(&[]); + for triple in [ + "arm64-apple-macosx", + "aarch64-apple-ios", + "aarch64-apple-watchos", + "arm64_32-apple-watchos", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "aarch64-linux-android", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + "i686-unknown-linux-gnu", + ] { + let t = TargetTriple::create(triple); + assert!( + Target::from_triple(&t).is_ok(), + "{triple} has no initialized LLVM backend — the compile driver \ + can emit this triple, so `global_init` must initialize it" + ); + } + } + + /// #7327 CI regression: an empty CPU string makes LLVM pick `generic`, + /// which on aarch64 is ARMv8.0 and has no FEAT_JSCVT — so the + /// `llvm.aarch64.fjcvtzs` that codegen emits for any Apple arm64 triple + /// cannot be selected and the compile aborts. Clang defaults that triple to + /// `apple-m1`, which is the assumption `set_jscvt_for_target` already makes. + /// Reproduced with `PERRY_TARGET_CPU=generic`, which is the path CI took. + #[test] + fn apple_aarch64_defaults_to_a_cpu_with_feat_jscvt() { + for triple in [ + "arm64-apple-macosx", + "arm64-apple-darwin", + "aarch64-apple-darwin", + "arm64-apple-ios", + ] { + assert_eq!( + default_cpu_for_triple(triple), + "apple-m1", + "{triple} must not fall back to LLVM's ARMv8.0 `generic`: codegen \ + emits llvm.aarch64.fjcvtzs for Apple arm64 triples" + ); + } + // Everything else keeps LLVM's portable baseline, matching the clang + // path when no tuning flag is passed. + for triple in [ + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ] { + assert_eq!(default_cpu_for_triple(triple), "", "{triple}"); + } + } + + /// `-S` used to be swallowed by the catch-all that ignores `-c`, so the + /// statepoint backends asked for assembly and were handed an object. The + /// failure was invisible here and surfaced two steps later as + /// `ld: unknown file type`, because #7314's compact-map rewriter rewrites + /// `.llvm_stackmaps` in assembly *text* and had nothing to rewrite. + #[test] + fn dash_s_requests_assembly_and_dash_c_does_not() { + let (_, _, _, _, emit_asm) = + interpret_plan_args(&["-O2".into(), "-S".into()]).expect("args parse"); + assert!(emit_asm, "-S must request assembly"); + + let (_, _, _, _, emit_asm) = + interpret_plan_args(&["-O2".into(), "-c".into()]).expect("args parse"); + assert!(!emit_asm, "-c must still request an object"); + } + + /// The property the wiring depends on: the same module emitted with + /// `FileType::Assembly` is assembler text carrying a stack-map section, + /// not an object. If this ever silently produced an object again, the + /// compact-map rewrite would find no `.llvm_stackmaps` to shrink and the + /// GC would be reading an empty map — the #7332 shape, a binary that + /// looks correct until a collection frees something live. + #[test] + fn assembly_emission_is_text_not_an_object() { + let context = Context::create(); + let ir = r#" +define i32 @f(i32 %x) { +entry: + %y = add i32 %x, 1 + ret i32 %y +} +"#; + let module = parse_ir_text(&context, ir, "asm_probe").expect("probe parses"); + global_init(&[]); + let triple = TargetMachine::get_default_triple(); + let target = Target::from_triple(&triple).expect("host target"); + let tm = target + .create_target_machine( + &triple, + "", + "", + OptimizationLevel::None, + RelocMode::PIC, + CodeModel::Default, + ) + .expect("target machine"); + + let asm = tm + .write_to_memory_buffer(&module, FileType::Assembly) + .expect("assembly emission"); + let text = String::from_utf8_lossy(asm.as_slice()).to_string(); + assert!( + text.contains(".globl") || text.contains(".global"), + "expected assembler directives, got:\n{}", + &text[..text.len().min(200)] + ); + + let obj = tm + .write_to_memory_buffer(&module, FileType::Object) + .expect("object emission"); + assert_ne!( + asm.as_slice(), + obj.as_slice(), + "assembly and object emission returned identical bytes — `-S` is \ + being ignored somewhere in the emission path" + ); + } + #[test] + fn tre_walk_budget_spellings() { + assert_eq!( + parse_tre_walk_budget(None), + TreWalkBudget::Cap(DEFAULT_TRE_MAX_ALLOCA_WALK) + ); + assert_eq!( + parse_tre_walk_budget(Some("")), + TreWalkBudget::Cap(DEFAULT_TRE_MAX_ALLOCA_WALK) + ); + assert_eq!(parse_tre_walk_budget(Some("0")), TreWalkBudget::Off); + assert_eq!(parse_tre_walk_budget(Some("off")), TreWalkBudget::Off); + assert_eq!(parse_tre_walk_budget(Some("false")), TreWalkBudget::Off); + assert_eq!( + parse_tre_walk_budget(Some(" 250000 ")), + TreWalkBudget::Cap(250_000) + ); + assert_eq!( + parse_tre_walk_budget(Some("lots")), + TreWalkBudget::Cap(DEFAULT_TRE_MAX_ALLOCA_WALK) + ); + } + + #[test] + fn fast_emit_budget_spellings() { + assert_eq!( + parse_fast_emit_budget(None), + FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS) + ); + assert_eq!( + parse_fast_emit_budget(Some("")), + FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS) + ); + assert_eq!(parse_fast_emit_budget(Some("0")), FastEmitBudget::Off); + assert_eq!(parse_fast_emit_budget(Some("off")), FastEmitBudget::Off); + assert_eq!(parse_fast_emit_budget(Some("false")), FastEmitBudget::Off); + assert_eq!( + parse_fast_emit_budget(Some(" 250000 ")), + FastEmitBudget::Cap(250_000) + ); + assert_eq!( + parse_fast_emit_budget(Some("lots")), + FastEmitBudget::Cap(DEFAULT_FAST_EMIT_MAX_INSTRS) + ); + } + + /// Two functions: `wide` has 4 allocas across 9 instructions (estimate + /// 36), `narrow` has one across 3 (estimate 3), and `decl` has no body. + fn alloca_walk_fixture() -> &'static str { + r#" +declare void @sink(ptr) + +define void @wide() { +entry: + %a = alloca i64 + %b = alloca i64 + %c = alloca i64 + %d = alloca i64 + call void @sink(ptr %a) + call void @sink(ptr %b) + call void @sink(ptr %c) + call void @sink(ptr %d) + ret void +} + +define void @narrow() { +entry: + %a = alloca i64 + call void @sink(ptr %a) + ret void +} +"# + } + + fn has_disable_tail_calls(module: &inkwell::module::Module<'_>, name: &str) -> bool { + module + .get_function(name) + .expect("fixture function exists") + .get_string_attribute( + inkwell::attributes::AttributeLoc::Function, + DISABLE_TAIL_CALLS_ATTR, + ) + .is_some_and(|attr| attr.get_string_value().to_bytes() == b"true") + } + + /// The budget is `allocas × instructions`, applied per function: only + /// the function over it is stamped, the boundary is exclusive, and + /// `off` stamps nothing. + #[test] + fn tre_budget_stamps_only_the_function_over_it() { + let context = Context::create(); + let module = parse_ir_text(&context, alloca_walk_fixture(), "tre_budget_fixture") + .expect("fixture parses"); + let wide = module.get_function("wide").expect("wide"); + let narrow = module.get_function("narrow").expect("narrow"); + assert_eq!(alloca_walk_factors(wide), (4, 9)); + assert_eq!(alloca_walk_factors(narrow), (1, 3)); + + assert!( + disable_tail_call_elim_over_budget(&module, TreWalkBudget::Off).is_empty(), + "a disabled budget stamps nothing" + ); + assert!(!has_disable_tail_calls(&module, "wide")); + + let exact = disable_tail_call_elim_over_budget(&module, TreWalkBudget::Cap(36)); + assert!(exact.is_empty(), "the cap is inclusive: {exact:?}"); + + let over = disable_tail_call_elim_over_budget(&module, TreWalkBudget::Cap(35)); + assert_eq!( + over, + vec![TreWalkOverBudget { + name: "wide".to_string(), + allocas: 4, + instructions: 9, + cap: 35, + }] + ); + assert!(has_disable_tail_calls(&module, "wide")); + assert!(!has_disable_tail_calls(&module, "narrow")); + let message = over[0].to_string(); + for needle in [ + "`wide`", + "4 allocas", + "9 instructions", + "estimate 36", + "budget 35", + "PERRY_LL_TRE_MAX_ALLOCA_WALK", + "#8883", + ] { + assert!( + message.contains(needle), + "{needle} missing from:\n{message}" + ); + } + assert!( + !message.contains("optnone"), + "the budget must never read as a demotion:\n{message}" + ); + } + + /// Selection is per function, the boundary is inclusive, declarations do + /// not count, and the diagnostic names the widest violating function. + #[test] + fn fast_emit_budget_selects_only_above_the_boundary() { + let context = Context::create(); + let module = parse_ir_text(&context, alloca_walk_fixture(), "fast_emit_fixture") + .expect("fixture parses"); + assert!(fast_emit_fallback(&module, FastEmitBudget::Off).is_none()); + assert!(fast_emit_fallback(&module, FastEmitBudget::Cap(9)).is_none()); + + let fallback = fast_emit_fallback(&module, FastEmitBudget::Cap(8)) + .expect("wide is one instruction over the budget"); + assert_eq!( + fallback, + FastEmitFallback { + name: "wide".to_string(), + instructions: 9, + cap: 8, + } + ); + let message = fallback.to_string(); + for needle in [ + "`wide`", + "9 instructions", + "budget 8", + "requested IR optimization", + "O0 machine pipeline", + "PERRY_LL_FAST_EMIT_MAX_INSTRS", + ] { + assert!( + message.contains(needle), + "{needle:?} missing from:\n{message}" + ); + } + } + + /// A self-recursive tail call that TailCallElim turns into a loop at + /// the pinned LLVM: with no attribute the recursive `call` disappears, + /// with `"disable-tail-calls"="true"` (exactly what the budget stamps) + /// it survives the full `default` pipeline — so the lever the + /// budget pulls is live, not merely spelled. + fn tail_recursive_fixture(attrs: &str) -> String { + format!( + "define i64 @count_down(i64 %n, i64 %acc) noinline {attrs} {{\n\ + entry:\n\ + \x20 %done = icmp eq i64 %n, 0\n\ + \x20 br i1 %done, label %ret, label %rec\n\ + rec:\n\ + \x20 %n1 = sub i64 %n, 1\n\ + \x20 %acc1 = add i64 %acc, %n\n\ + \x20 %r = call i64 @count_down(i64 %n1, i64 %acc1)\n\ + \x20 ret i64 %r\n\ + ret:\n\ + \x20 ret i64 %acc\n\ + }}\n" + ) + } + + #[test] + fn disable_tail_calls_attribute_stops_tail_call_elim_at_the_pinned_llvm() { + let target = crate::codegen::default_target_triple(); + let with_tre = statepoint_rewritten_ir_with_passes( + &tail_recursive_fixture(""), + &target, + "tre_control", + "default", + ) + .expect("control optimizes"); + assert!( + !with_tre.contains("call i64 @count_down"), + "control: TailCallElim must turn the tail recursion into a loop, or this test \ + cannot tell the attribute apart from a no-op:\n{with_tre}" + ); + + let without_tre = statepoint_rewritten_ir_with_passes( + &tail_recursive_fixture(&format!("\"{DISABLE_TAIL_CALLS_ATTR}\"=\"true\"")), + &target, + "tre_disabled", + "default", + ) + .expect("attributed fixture optimizes"); + assert!( + without_tre.contains("call i64 @count_down"), + "the attribute must keep TailCallElim off the function:\n{without_tre}" + ); + } + + /// The budget is wired into the shipped emission path: under a cap of + /// zero every function with an alloca is stamped before `default` + /// runs, the per-unit stats name it, and the unit still emits. + #[test] + fn tre_budget_is_applied_by_the_shipped_pipeline() { + global_init(&[]); + let target = crate::codegen::default_target_triple(); + let context = Context::create(); + let module = parse_ir_text(&context, alloca_walk_fixture(), "tre_budget_shipped") + .expect("fixture parses"); + let mut stats = UnitCodegenStats::default(); + let object = with_test_tre_walk_budget(0, || { + optimize_and_emit_module_with_stats( + &module, + &target, + &["-Os".into(), "-c".into()], + false, + Some(&mut stats), + ) + }) + .expect("a stamped module still optimizes and emits"); + assert!(!object.is_empty()); + let mut names: Vec<&str> = stats + .tail_call_elim_skipped + .iter() + .map(|over| over.name.as_str()) + .collect(); + names.sort_unstable(); + assert_eq!(names, ["narrow", "wide"]); + + // -O0 runs no TailCallElim, so nothing is stamped there. + let module = parse_ir_text(&context, alloca_walk_fixture(), "tre_budget_o0") + .expect("fixture parses"); + let mut stats = UnitCodegenStats::default(); + with_test_tre_walk_budget(0, || { + optimize_and_emit_module_with_stats( + &module, + &target, + &["-O0".into(), "-c".into()], + false, + Some(&mut stats), + ) + }) + .expect("-O0 emits"); + assert!(stats.tail_call_elim_skipped.is_empty()); + assert!(!has_disable_tail_calls(&module, "wide")); + } + + /// A tiny test cap proves the shipped path records and successfully uses + /// the second, O0 target machine only after running the requested Os IR + /// pipeline. The production threshold is pinned by the parser test and + /// the real Claude-Code measurement in its constant's documentation. + #[test] + fn fast_emit_budget_is_applied_by_the_shipped_pipeline() { + global_init(&[]); + let target = crate::codegen::default_target_triple(); + let context = Context::create(); + let module = parse_ir_text(&context, alloca_walk_fixture(), "fast_emit_shipped") + .expect("fixture parses"); + let mut stats = UnitCodegenStats::default(); + let object = with_test_fast_emit_budget(1, || { + optimize_and_emit_module_with_stats( + &module, + &target, + &["-Os".into(), "-c".into()], + false, + Some(&mut stats), + ) + }) + .expect("the already-optimized module emits through the bounded target machine"); + assert!(!object.is_empty()); + let fallback = stats + .fast_emit_fallback + .expect("the shipped path must report the selected fallback"); + assert_eq!(fallback.name, "wide"); + assert!(fallback.instructions > fallback.cap); + assert_eq!(fallback.cap, 1); + + // A requested O0 compile already uses the bounded target machine; it + // neither needs nor reports a fallback. + let module = + parse_ir_text(&context, alloca_walk_fixture(), "fast_emit_o0").expect("fixture parses"); + let mut stats = UnitCodegenStats::default(); + with_test_fast_emit_budget(1, || { + optimize_and_emit_module_with_stats( + &module, + &target, + &["-O0".into(), "-c".into()], + false, + Some(&mut stats), + ) + }) + .expect("-O0 emits"); + assert!(stats.fast_emit_fallback.is_none()); + } +} diff --git a/crates/perry-codegen/src/lib.rs b/crates/perry-codegen/src/lib.rs index af644d78bb..3a676a2d0b 100644 --- a/crates/perry-codegen/src/lib.rs +++ b/crates/perry-codegen/src/lib.rs @@ -74,10 +74,10 @@ pub mod types; pub use codegen::{ compile_module, namespace_member_class_key, namespace_member_func_key, - namespace_member_var_key, resolve_target_triple, short_spread_method_capabilities, AppMetadata, - CompileOptions, ExportedObjectLiteralCapability, FpContractMode, ImportedClass, - ImportedObjectLiteral, ImportedObjectLiteralMethod, NamespaceEntry, NamespaceEntryKind, - ObjectLiteralMethodCandidate, ShortSpreadMethodCandidate, + namespace_member_var_key, resolve_target_triple, short_spread_method_capabilities, + user_function_symbol, AppMetadata, CompileOptions, ExportedObjectLiteralCapability, + FpContractMode, ImportedClass, ImportedObjectLiteral, ImportedObjectLiteralMethod, + NamespaceEntry, NamespaceEntryKind, ObjectLiteralMethodCandidate, ShortSpreadMethodCandidate, }; pub use collectors::CjsPreambleCensus; diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index ff6804db34..a1dbc6b192 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -342,6 +342,31 @@ fn size_optimization_requested(value: Option<&str>) -> bool { } } +/// Select the LLVM optimization level for generated application modules. +/// +/// Normal builds retain Perry's measured `-Os` default (or the existing +/// `PERRY_LL_SIZE_OPT=0` opt-out to `-O3`). `PERRY_LL_OPT_LEVEL` is an explicit +/// diagnostic/build-through override for dependency bundles whose generated +/// functions are too large for a useful optimized build. It accepts the same +/// level spellings as clang and deliberately leaves an unrecognized value on +/// the normal default instead of silently disabling optimization. +fn application_opt_flag(explicit: Option<&str>, size_opt: Option<&str>) -> &'static str { + match explicit + .map(str::trim) + .map(str::to_ascii_lowercase) + .as_deref() + { + Some("0" | "o0") => "-O0", + Some("1" | "o1") => "-O1", + Some("2" | "o2") => "-O2", + Some("3" | "o3") => "-O3", + Some("s" | "os") => "-Os", + Some("z" | "oz") => "-Oz", + _ if size_optimization_requested(size_opt) => "-Os", + _ => "-O3", + } +} + fn build_clang_compile_plan( clang: PathBuf, ll_path: PathBuf, @@ -361,13 +386,11 @@ fn build_clang_compile_plan( // Perry defaults to SIZE-optimized native output: `-Os` measured no runtime // cost on the benchmark corpus (see `size_optimization_requested`), and it // materially shrinks dense generated bundles. `PERRY_LL_SIZE_OPT=0` restores - // `-O3`. There is no module-size-driven policy change. + // `-O3`; the explicit `PERRY_LL_OPT_LEVEL` override wins over both. There is + // no implicit module-size-driven policy change. let size_opt = env::var("PERRY_LL_SIZE_OPT").ok(); - let opt_flag = if size_optimization_requested(size_opt.as_deref()) { - "-Os" - } else { - "-O3" - }; + let explicit_opt = env::var("PERRY_LL_OPT_LEVEL").ok(); + let opt_flag = application_opt_flag(explicit_opt.as_deref(), size_opt.as_deref()); // Compacting the stack map means going through assembly, because that is // where LLVM prints the map's function addresses as symbol *names* — the diff --git a/crates/perry-codegen/src/linker_tests.rs b/crates/perry-codegen/src/linker_tests.rs index dfbd0a6209..06500f0201 100644 --- a/crates/perry-codegen/src/linker_tests.rs +++ b/crates/perry-codegen/src/linker_tests.rs @@ -223,6 +223,23 @@ fn size_optimization_is_on_unless_explicitly_disabled() { } } +#[test] +fn explicit_application_opt_level_overrides_the_size_default() { + for (spelling, expected) in [ + ("0", "-O0"), + ("o0", "-O0"), + ("1", "-O1"), + ("O2", "-O2"), + ("3", "-O3"), + ("s", "-Os"), + ("Oz", "-Oz"), + ] { + assert_eq!(application_opt_flag(Some(spelling), None), expected); + } + assert_eq!(application_opt_flag(Some("unknown"), None), "-Os"); + assert_eq!(application_opt_flag(Some("unknown"), Some("0")), "-O3"); +} + #[test] fn compile_plan_skips_native_tuning_for_explicit_target() { let plan = build_clang_compile_plan( diff --git a/crates/perry-codegen/src/lower_call/new.rs b/crates/perry-codegen/src/lower_call/new.rs index fdc93a03c5..fade7668bc 100644 --- a/crates/perry-codegen/src/lower_call/new.rs +++ b/crates/perry-codegen/src/lower_call/new.rs @@ -511,6 +511,18 @@ fn lower_new_impl_inner<'a>( let mut lowered_args: Vec = Vec::with_capacity(args.len()); for a in args { let value = lower_constructor_arg(ctx, a)?; + // An argument can complete abruptly while still returning a sentinel + // value to the lowering API. The unresolved dynamic-Worker fallback + // is one such expression: it emits the runtime throw followed by + // `unreachable`. Do not root that sentinel or continue into instance + // allocation / constructor diamonds. `LlBlock` drops instructions + // appended after a terminator, while those diamonds create fresh + // blocks that would refer to the dropped registers (Claude Code's + // `{ worker: new Worker(dynamicPath), stamp: ... }` object literal was + // the reproducer). + if ctx.block().is_terminated() { + return Ok(double_literal(f64::from_bits(crate::nanbox::TAG_UNDEFINED))); + } // `collects` is unconditionally true: the instance allocation below // always collects, so every argument is live across it. That is the // same answer the pre-migration code gave by consulting diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index 4839d3264e..74807711eb 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -702,8 +702,8 @@ impl LlModule { /// `external` *declarations* are replicated as-is; /// * the module's external `declare`s plus a synthesized `declare` for /// every locally-defined function the unit does NOT itself define, so - /// cross-unit calls resolve at link time (deduped by name, existing - /// declarations win); + /// cross-unit calls resolve at link time (deduped by name, local + /// definitions supply the authoritative signature); /// * each function rendered with external linkage forced (the lone /// `internal` init/wrapper is promoted so cross-unit calls bind); /// * the shared attribute groups + metadata (so `#N`/`!N` refs resolve). @@ -759,18 +759,19 @@ impl LlModule { let shared_strings: Vec = self.string_constants.clone(); let shared_globals: Vec = self.globals.clone(); - // name -> declare line. Existing module declarations (runtime, FFI, - // cross-module) take precedence; every locally-defined function without - // one gets a synthesized declare. Deduped by name so no unit emits a - // duplicate declaration. BTreeMap for deterministic unit output. + // name -> declare line. Start with module declarations (runtime, FFI, + // cross-module), then replace any entry that is also defined locally + // with a declaration synthesized from that definition. Import metadata + // can contain an earlier, less precise signature; the definition is what + // the whole-module renderer and LLVM see, so split units must agree with + // it too. Deduped by name so no unit emits a duplicate declaration. + // BTreeMap keeps unit output deterministic. let mut decl_by_name: BTreeMap<&str, String> = BTreeMap::new(); for (name, decl) in &self.declarations { decl_by_name.insert(name.as_str(), decl.clone()); } for f in &funcs { - decl_by_name - .entry(f.name.as_str()) - .or_insert_with(|| declare_line_for(f)); + decl_by_name.insert(f.name.as_str(), declare_line_for(f)); } // #7174 (real-app scaling): scan each bucket's functions first, then @@ -1769,6 +1770,74 @@ mod tests { assert!(ir.contains("define i32 @main")); } + #[test] + fn split_unit_declaration_uses_local_definition_signature() { + let mut m = LlModule::new("arm64-apple-macosx15.0.0"); + + // Import metadata may register a constructor before its source module + // is lowered, with a stale arity. Once this module defines the symbol, + // its definition is authoritative for callers placed in another unit. + m.declare_function("constructor", DOUBLE, &[DOUBLE]); + let constructor = m.define_function( + "constructor", + DOUBLE, + vec![ + (DOUBLE, "this_arg".into()), + (DOUBLE, "arg0".into()), + (DOUBLE, "arg1".into()), + ], + ); + constructor.create_block("entry").ret(DOUBLE, "this_arg"); + + let caller = m.define_function("caller", DOUBLE, vec![]); + let entry = caller.create_block("entry"); + let result = entry.call( + DOUBLE, + "constructor", + &[(DOUBLE, "0.0"), (DOUBLE, "1.0"), (DOUBLE, "2.0")], + ); + entry.ret(DOUBLE, &result); + + let units = m.render_codegen_units(2); + let caller_unit = units + .iter() + .find(|unit| unit.contains("define double @caller(")) + .expect("caller unit"); + assert!(caller_unit.contains("declare double @constructor(double, double, double)")); + assert!(!caller_unit.contains("declare double @constructor(double)")); + } + + #[test] + fn split_unit_declares_local_function_used_as_pointer_argument() { + let mut m = LlModule::new("arm64-apple-macosx15.0.0"); + m.declare_function("js_closure_alloc_singleton", I64, &[PTR]); + + let wrapper_name = "__perry_wrap_perry_fn_m___a"; + let wrapper = m.define_function( + wrapper_name, + DOUBLE, + vec![(I64, "%this_closure".into()), (DOUBLE, "%a0".into())], + ); + wrapper.create_block("entry").ret(DOUBLE, "%a0"); + + let init = m.define_function("m__init_body", VOID, vec![]); + let entry = init.create_block("entry"); + entry.call( + I64, + "js_closure_alloc_singleton", + &[(PTR, &format!("@{wrapper_name}"))], + ); + entry.ret_void(); + + let units = m.render_codegen_units(2); + let init_unit = units + .iter() + .find(|unit| unit.contains("define void @m__init_body(")) + .expect("init unit"); + assert!(!init_unit.contains(&format!("define double @{wrapper_name}("))); + assert!(init_unit.contains(&format!("declare double @{wrapper_name}(i64, double)"))); + } + #[test] fn string_constant_escapes_nonprintable() { let mut m = LlModule::new("arm64-apple-macosx15.0.0"); diff --git a/crates/perry-codegen/src/native_emit.rs b/crates/perry-codegen/src/native_emit.rs index 8edeb59bfa..85932894f8 100644 --- a/crates/perry-codegen/src/native_emit.rs +++ b/crates/perry-codegen/src/native_emit.rs @@ -169,7 +169,7 @@ struct FrozenUnit { function_count: usize, } -/// Apply a typed post-RS4GC budget request to the lowering-owned functions +/// Apply a typed pre- or post-RS4GC budget request to the lowering-owned functions /// that produced a module/unit. The request is expected to make progress for /// every named function; otherwise retrying would either preserve the refusal /// or loop forever, so fail with the original names and counts instead. @@ -187,17 +187,37 @@ pub(crate) fn apply_budget_spill_retry<'a>( }; if function.request_shadow_frame_spill() { changed.insert(violation.name.clone()); - eprintln!( - "perry: `{}` exceeded the post-RS4GC instruction budget ({} -> {} \ - instructions; cap {}); retrying it with precise GC roots in a shadow \ - frame at the requested optimization level (#8679)", - violation.name, - violation - .pre_instructions - .map_or_else(|| "unknown".to_string(), |n| n.to_string()), - violation.post_instructions, - violation.cap, - ); + match &violation.cause { + crate::inprocess::Rs4gcBudgetCause::PreRewrite { + root_allocas, + safepoints, + estimated_relocations, + } => eprintln!( + "perry: `{}` exceeded the pre-RS4GC relocation estimate ({} managed-root \ + allocas + {} non-leaf call-result temporaries across {} call sites = {} \ + estimated relocations; cap {}); retrying it with precise GC roots in a \ + shadow frame at the requested optimization level (#8583)", + violation.name, + root_allocas, + safepoints, + safepoints, + estimated_relocations, + violation.cap, + ), + crate::inprocess::Rs4gcBudgetCause::PostRewrite { post_instructions } => { + eprintln!( + "perry: `{}` exceeded the post-RS4GC instruction budget ({} -> {} \ + instructions; cap {}); retrying it with precise GC roots in a shadow \ + frame at the requested optimization level (#8679)", + violation.name, + violation + .pre_instructions + .map_or_else(|| "unknown".to_string(), |n| n.to_string()), + post_instructions, + violation.cap, + ); + } + } } } let missing: Vec<&str> = violations @@ -209,7 +229,7 @@ pub(crate) fn apply_budget_spill_retry<'a>( Ok(()) } else { Err(anyhow!( - "post-RS4GC budget requested a shadow-frame retry for {}, but those \ + "RS4GC budget requested a shadow-frame retry for {}, but those \ functions were not available for a new lowering (or were already retried)", missing.join(", ") )) @@ -421,7 +441,7 @@ pub fn compile_module_units_native( let target_triple = llmod.target_triple.clone(); let owned_module = std::mem::replace(llmod, LlModule::new(target_triple)); // Keep at most a bounded window of lowering-owned units alive after they - // are frozen. A post-RS4GC budget miss needs that source graph exactly + // are frozen. A pre- or post-RS4GC budget miss needs that source graph exactly // once so the named functions can switch root lowering and be frozen // again; successful units are still dropped immediately (#8679). let mut parts: Vec> = owned_module @@ -671,6 +691,14 @@ pub fn compile_module_units_native( } } } else { + if show_progress { + eprintln!( + "[perry] codegen: {module_prefix}: LLVM unit {}/{} failed after {:.1}s: {error:#}", + i + 1, + unit_total, + attempt_elapsed.as_secs_f64() + ); + } slots[i] = Some(out); } } else { diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index 5772017e36..b14de19aa9 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -7346,6 +7346,93 @@ fn abrupt_captured_local_assignment_does_not_emit_orphan_write_barrier() { ); } +#[test] +fn abrupt_constructor_argument_stops_anonymous_object_construction() { + // Closed object literals are `new __AnonShape_*(field0, field1, ...)` by + // the time codegen sees them. Claude Code returns an object whose first + // field constructs an unresolved dynamic Worker and whose second field + // constructs an Int32Array. The Worker emits throw + unreachable, so the + // later field, allocation and constructor diamond must not be emitted: + // their definitions would be dropped from the terminated block while the + // newly-created blocks still used their registers. + let mut record = class( + 80, + "__AnonShape_abrupt_constructor_arg", + vec![ + class_field("worker", Type::Any), + class_field("stamp", Type::Any), + ], + ); + record.constructor = Some(Function { + id: 81, + name: "__AnonShape_abrupt_constructor_arg_constructor".to_string(), + type_params: Vec::new(), + params: vec![ + param(82, "worker", Type::Any), + param(83, "stamp", Type::Any), + ], + return_type: Type::Any, + body: vec![ + Stmt::Expr(Expr::PropertySet { + object: Box::new(Expr::This), + property: "worker".to_string(), + value: Box::new(local(82)), + }), + Stmt::Expr(Expr::PropertySet { + object: Box::new(Expr::This), + property: "stamp".to_string(), + value: Box::new(local(83)), + }), + ], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }); + let module = module_with_classes_and_params( + "abrupt_anonymous_object_constructor_arg.ts", + vec![record], + vec![param(99, "filename", Type::Any)], + Type::Any, + vec![Stmt::Return(Some(Expr::New { + class_name: "__AnonShape_abrupt_constructor_arg".to_string(), + args: vec![ + Expr::WorkerNew { + paths: Vec::new(), + filename: Box::new(local(99)), + options: None, + is_eval: false, + }, + Expr::Array(Vec::new()), + ], + type_args: Vec::new(), + byte_offset: 0, + cap_args_appended: 0, + }))], + ); + let ir = String::from_utf8(compile_module(&module, empty_opts()).unwrap()).unwrap(); + let body = probe_body(&ir); + let throw = body + .find("call void @js_throw_error_with_code") + .expect("unresolved Worker construction should emit its runtime throw"); + let after_throw = &body[throw..]; + + assert!( + after_throw.contains("\n unreachable"), + "the dynamic Worker fallback must terminate the path:\n{after_throw}" + ); + assert!( + !after_throw.contains("js_array_alloc") + && !after_throw.contains("js_object_alloc") + && !after_throw.contains("ctor_prologue"), + "nothing after an abruptly-completing constructor argument may be lowered:\n{after_throw}" + ); +} + fn boxed_param_capture_module(name: &str) -> Module { module_with_classes_and_params( name, diff --git a/crates/perry-ext-http/src/test_async_shims.rs b/crates/perry-ext-http/src/test_async_shims.rs index d6bd8c9f12..a2a6f2146c 100644 --- a/crates/perry-ext-http/src/test_async_shims.rs +++ b/crates/perry-ext-http/src/test_async_shims.rs @@ -68,5 +68,8 @@ pub extern "C" fn perry_ffi_spawn_async(_ctx: *mut c_void) {} // Linking the ws dispatch extension also retains its synchronous polling // helper. These unit tests use the no-op task shim above; real networking is // exercised by the compiled HTTP/WebSocket integration tests. +// Same story: `js_bun_tcp_listen` drives the shared runtime from its bind-poll +// loop (`perry_ffi::run_pending`), so linking perry-ext-net's object code into +// this crate's test binaries pulls the extern in with it. #[no_mangle] pub extern "C" fn perry_ffi_run_pending(_budget_ms: u64) {} diff --git a/crates/perry-hir/src/lower/tests.rs b/crates/perry-hir/src/lower/tests.rs index 54f2be4089..d13fbea68b 100644 --- a/crates/perry-hir/src/lower/tests.rs +++ b/crates/perry-hir/src/lower/tests.rs @@ -984,93 +984,6 @@ fn test_perry_ui_widget_factory_handle_classification() { assert!(!super::perry_ui_factory_returns_handle("widgetAddChild")); } -/// #6642: native lowering must preserve the Widget compatibility methods on -/// factory results and explicitly Widget-typed parameters. -#[test] -fn test_perry_ui_widget_add_child_uses_native_dispatch() { - use crate::ir::{clear_current_module_source, Expr, Stmt}; - - let source = r#" - import { VStack, Text, type Widget } from "perry/ui"; - - const parent = VStack(0, []); - const child = Text("hello"); - parent.addChild(child); - parent.removeAllChildren(); - - function attach(target: Widget, item: Widget) { - target.addChild(item); - } - "#; - let module = - perry_parser::parse_typescript(source, "widget_add_child.ts").expect("source should parse"); - let hir = - super::lower_module(&module, "test", "widget_add_child.ts").expect("source should lower"); - clear_current_module_source(); - - let call = hir.init.iter().find_map(|stmt| match stmt { - Stmt::Expr(Expr::NativeMethodCall { - module, - class_name, - object, - method, - args, - }) if method == "addChild" => Some((module, class_name, object, args)), - _ => None, - }); - - assert!( - matches!( - call, - Some((module, Some(class_name), Some(_), args)) - if module == "perry/ui" && class_name == "VStack" && args.len() == 1 - ), - "Widget.addChild must lower as a perry/ui instance call, got: {:#?}", - hir.init - ); - - assert!( - hir.init.iter().any(|stmt| matches!( - stmt, - Stmt::Expr(Expr::NativeMethodCall { - module, - class_name: Some(class_name), - object: Some(_), - method, - args, - }) if module == "perry/ui" - && class_name == "VStack" - && method == "removeAllChildren" - && args.is_empty() - )), - "Widget.removeAllChildren must lower as a perry/ui instance call, got: {:#?}", - hir.init - ); - - let attach = hir - .functions - .iter() - .find(|function| function.name == "attach") - .expect("attach should lower"); - assert!( - attach.body.iter().any(|stmt| matches!( - stmt, - Stmt::Expr(Expr::NativeMethodCall { - module, - class_name: Some(class_name), - object: Some(_), - method, - args, - }) if module == "perry/ui" - && class_name == "Widget" - && method == "addChild" - && args.len() == 1 - )), - "Widget-typed parameters must use perry/ui instance dispatch, got: {:#?}", - attach.body - ); -} - /// #6679: a NAMED class EXPRESSION's `.name` is its own explicit name /// (`Named` in `const B = class Named {}`), not the outer binding name. Per /// spec a named class expression is not an anonymous function definition, so @@ -1707,6 +1620,30 @@ fn test_create_require_local_keeps_the_native_namespace_fast_path() { ); } +/// Bun exposes a synchronous module loader as `import.meta.require`. It must +/// share Perry's synchronous dynamic-require path; the generic import.meta +/// member lowering intentionally maps unknown properties to `undefined`. +#[test] +fn import_meta_require_lowers_to_synchronous_module_dispatch() { + let source = r#" + const direct = import.meta.require("/$bunfs/root/chunk-a.js"); + const computed = import.meta["require"]("./chunk-b.js"); + console.log(direct, computed); + "#; + let module = perry_parser::parse_typescript(source, "t.ts").expect("source parses"); + let hir = super::lower_module(&module, "t", "t.ts").expect("source lowers"); + let dump = format!("{hir:#?}"); + assert_eq!( + dump.matches("synchronous: true").count(), + 2, + "both import.meta.require spellings must use synchronous module dispatch: {dump}" + ); + assert!( + dump.contains("/$bunfs/root/chunk-a.js") && dump.contains("./chunk-b.js"), + "the original specifiers must reach the module collector: {dump}" + ); +} + /// The #8465 counterpart, complementary to /// `test_user_require_function_with_body_still_shadows_the_intrinsic` above: /// that one pins that a real `function require` body suppresses the fold; this @@ -1983,3 +1920,4 @@ mod capture_stash; mod mixin_parent_chain; mod nullish_over_optional_chain; +mod ui_widget_add_child; diff --git a/crates/perry-hir/src/lower/tests/ui_widget_add_child.rs b/crates/perry-hir/src/lower/tests/ui_widget_add_child.rs new file mode 100644 index 0000000000..80c268fd0a --- /dev/null +++ b/crates/perry-hir/src/lower/tests/ui_widget_add_child.rs @@ -0,0 +1,91 @@ +//! #6642: `perry/ui` widget lowering must preserve the Widget compatibility +//! of factory results and explicitly Widget-typed parameters. +//! +//! Split out of `lower/tests.rs` to keep it under the 2000-line size gate. + +/// #6642: native lowering must preserve the Widget compatibility methods on +/// factory results and explicitly Widget-typed parameters. +#[test] +fn test_perry_ui_widget_add_child_uses_native_dispatch() { + use crate::ir::{clear_current_module_source, Expr, Stmt}; + + let source = r#" + import { VStack, Text, type Widget } from "perry/ui"; + + const parent = VStack(0, []); + const child = Text("hello"); + parent.addChild(child); + parent.removeAllChildren(); + + function attach(target: Widget, item: Widget) { + target.addChild(item); + } + "#; + let module = + perry_parser::parse_typescript(source, "widget_add_child.ts").expect("source should parse"); + let hir = + super::lower_module(&module, "test", "widget_add_child.ts").expect("source should lower"); + clear_current_module_source(); + + let call = hir.init.iter().find_map(|stmt| match stmt { + Stmt::Expr(Expr::NativeMethodCall { + module, + class_name, + object, + method, + args, + }) if method == "addChild" => Some((module, class_name, object, args)), + _ => None, + }); + + assert!( + matches!( + call, + Some((module, Some(class_name), Some(_), args)) + if module == "perry/ui" && class_name == "VStack" && args.len() == 1 + ), + "Widget.addChild must lower as a perry/ui instance call, got: {:#?}", + hir.init + ); + + assert!( + hir.init.iter().any(|stmt| matches!( + stmt, + Stmt::Expr(Expr::NativeMethodCall { + module, + class_name: Some(class_name), + object: Some(_), + method, + args, + }) if module == "perry/ui" + && class_name == "VStack" + && method == "removeAllChildren" + && args.is_empty() + )), + "Widget.removeAllChildren must lower as a perry/ui instance call, got: {:#?}", + hir.init + ); + + let attach = hir + .functions + .iter() + .find(|function| function.name == "attach") + .expect("attach should lower"); + assert!( + attach.body.iter().any(|stmt| matches!( + stmt, + Stmt::Expr(Expr::NativeMethodCall { + module, + class_name: Some(class_name), + object: Some(_), + method, + args, + }) if module == "perry/ui" + && class_name == "Widget" + && method == "addChild" + && args.len() == 1 + )), + "Widget-typed parameters must use perry/ui instance dispatch, got: {:#?}", + attach.body + ); +} diff --git a/crates/perry-runtime/src/arena/alloc_sample.rs b/crates/perry-runtime/src/arena/alloc_sample.rs new file mode 100644 index 0000000000..36d69ca059 --- /dev/null +++ b/crates/perry-runtime/src/arena/alloc_sample.rs @@ -0,0 +1,269 @@ +//! `PERRY_ALLOC_SITE_SAMPLE=`: byte-proportional allocation-site +//! sampling for the GC arena — WHAT allocates, by call chain and object type. +//! +//! The question it answers: a 3,300-character streamed reply in the compiled +//! claude-code TUI pushes ~890 MB through the nursery (86 copying minors) +//! while node allocates a small fraction of that for the same interaction. +//! Nothing in the runtime could say which JS operation, or which runtime +//! helper under it, produced the volume. This does, the way V8's sampling +//! heap profiler does: every `` of arena allocation, capture the +//! native return-address chain of the allocation that crossed the boundary. +//! Each sample stands for `` of allocation, so a site's share of the +//! samples is its share of the bytes, independent of its object size mix. +//! +//! Coverage: +//! +//! * every runtime allocation path — [`super::arena_alloc_gc`], +//! `arena_alloc_gc_no_collect`, the old-gen births, the longlived arena — +//! decrements a per-thread countdown (one relaxed atomic load when the +//! sampler is off, the only cost the default build pays); +//! * the codegen inline bump allocator never enters the runtime, so while +//! sampling is on the mirrored `InlineArenaState.size` is capped at +//! `offset + ` ([`inline_limit`]), and +//! every site that writes the inline offset back to its block charges the +//! inline bytes allocated since the last sync to the SAME countdown +//! ([`note_inline_sync`]). One countdown for both paths is what makes the +//! weighting exact: capping at `offset + interval` on every resync (the +//! first cut) let a loop that interleaves runtime and inline allocations +//! push the cap ahead forever — 29 samples for 29 MB of inline objects. +//! Inert when off — the cap is the real block size. +//! +//! Report: `[alloc-site] …` lines after each copying minor and at process +//! exit — totals, bytes by object type, and the top sites as an +//! innermost-first chain resolved to JS display names where a frame is +//! compiled user code (`crate::error::describe_chain`), else the linker +//! symbol. Cumulative since process start. + +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Sampling interval in bytes; 0 = off. Written once at `gc_init`. +static INTERVAL: AtomicUsize = AtomicUsize::new(0); + +/// The interval a bare `=1`/`on` selects; pinned by the knob test. +pub(crate) const DEFAULT_INTERVAL_BYTES: usize = 64 * 1024; +const DEPTH: usize = 6; +const TYPE_SLOTS: usize = 32; + +#[derive(Default, Clone)] +struct Site { + samples: u64, + sampled_bytes: u64, + by_type: [u32; TYPE_SLOTS], +} + +#[derive(Default)] +struct Table { + sites: HashMap<[usize; DEPTH], Site>, + samples: u64, + sampled_bytes: u64, + by_type: [u64; TYPE_SLOTS], + inline_trips: u64, +} + +crate::perry_thread_local! { + static UNTIL: Cell = const { Cell::new(0) }; + static TABLE: RefCell = RefCell::new(Table::default()); +} + +/// Read `PERRY_ALLOC_SITE_SAMPLE` once (from `gc_init`). A bare `1` or an +/// unparsable value selects the default interval. +pub(crate) fn init_from_env() { + let raw = std::env::var("PERRY_ALLOC_SITE_SAMPLE").ok(); + let interval = parse_interval(raw.as_deref()); + if interval == 0 { + return; + } + INTERVAL.store(interval, Ordering::Relaxed); + eprintln!("[alloc-site] sampling every {interval} bytes of arena allocation"); +} + +/// The knob's value semantics, as a pure function so the OFF state can be +/// pinned by `gc/tests/env_knob_parse.rs` without touching the process +/// environment (the shared GC-knob vocabulary, #7991): the boolean spellings +/// read through [`crate::gc::env_flag_from_value`] — `1`/`on`/`true`/`yes` +/// select [`DEFAULT_INTERVAL_BYTES`], every OFF spelling and every typo read +/// as OFF; an integer ≥ 2 is the interval in bytes, floored at +/// [`MIN_INTERVAL_BYTES`] so a stray small value cannot turn every allocation +/// into a stack walk. +pub(crate) fn parse_interval(raw: Option<&str>) -> usize { + if crate::gc::env_flag_from_value(raw) { + return DEFAULT_INTERVAL_BYTES; + } + raw.and_then(|r| r.trim().parse::().ok()) + .filter(|&v| v >= 2) + .map_or(0, |v| v.max(MIN_INTERVAL_BYTES)) +} + +/// Smallest interval an explicit integer can select. +pub(crate) const MIN_INTERVAL_BYTES: usize = 256; + +/// A runtime-path allocation of `total` bytes (header included) of +/// `obj_type` is about to happen. +#[inline(always)] +pub(crate) fn note(total: usize, obj_type: u8) { + let interval = INTERVAL.load(Ordering::Relaxed); + if interval == 0 { + return; + } + note_slow(total, obj_type, interval); +} + +/// Charge `bytes` to the countdown; true when a sample is due (the countdown +/// is then re-armed with a full interval). +#[inline] +fn countdown(bytes: usize, interval: usize) -> bool { + UNTIL.with(|u| { + let left = u.get(); + if left > bytes { + u.set(left - bytes); + false + } else { + u.set(interval); + true + } + }) +} + +#[cold] +#[inline(never)] +fn note_slow(total: usize, obj_type: u8, interval: usize) { + if countdown(total, interval) { + sample(total, obj_type, false); + } +} + +/// A site is writing the inline bump offset back to its arena block: +/// `inline_offset - block_offset` bytes were allocated by the compiled fast +/// path since the last sync. Charge them to the shared countdown. +#[inline(always)] +pub(crate) fn note_inline_sync(block_offset: usize, inline_offset: usize) { + let interval = INTERVAL.load(Ordering::Relaxed); + if interval == 0 || inline_offset <= block_offset { + return; + } + note_inline_slow(inline_offset - block_offset, interval); +} + +#[cold] +#[inline(never)] +fn note_inline_slow(bytes: usize, interval: usize) { + if countdown(bytes, interval) { + // The inline allocator only births class instances (`GC_TYPE_OBJECT` + // with a per-site header image); the size charged is the whole burst. + sample(bytes, crate::gc::GC_TYPE_OBJECT, true); + } +} + +/// While sampling, cap the mirrored inline block limit at the bytes left +/// before the next sample, so the compiled fast path returns to the runtime +/// (`js_inline_arena_slow_alloc`, whose write-back charges the burst) exactly +/// when a sample is due. Identity when off. +#[inline(always)] +pub(crate) fn inline_limit(offset: usize, block_size: usize) -> usize { + let interval = INTERVAL.load(Ordering::Relaxed); + if interval == 0 { + return block_size; + } + let left = UNTIL.with(Cell::get).max(1); + block_size.min(offset.saturating_add(left)) +} + +fn sample(total: usize, obj_type: u8, inline_trip: bool) { + let mut pcs = [0usize; crate::error::MAX_CAPTURED_FRAMES]; + let n = crate::error::capture_ips(&mut pcs); + // Frame 0 is the return into this sampler; frame 1 is the allocation + // helper (or, on the inline path, the write-back site), and the chain + // walks out to the compiled JS function that owns the allocation. + let mut key = [0usize; DEPTH]; + for (slot, pc) in key.iter_mut().zip(&pcs[1.min(n)..n]) { + *slot = *pc; + } + let t = (obj_type as usize).min(TYPE_SLOTS - 1); + TABLE.with(|table| { + let Ok(mut table) = table.try_borrow_mut() else { + return; + }; + table.samples += 1; + table.sampled_bytes += total as u64; + table.by_type[t] += 1; + if inline_trip { + table.inline_trips += 1; + } + let site = table.sites.entry(key).or_default(); + site.samples += 1; + site.sampled_bytes += total as u64; + site.by_type[t] += 1; + }); +} + +fn type_name(t: usize) -> &'static str { + crate::gc::gc_type_info(t as u8).map_or("?", |i| i.name) +} + +/// Print the cumulative histogram. `label` names the occasion. +pub(crate) fn report(label: &str) { + let interval = INTERVAL.load(Ordering::Relaxed); + if interval == 0 { + return; + } + TABLE.with(|table| { + let Ok(table) = table.try_borrow() else { + return; + }; + if table.samples == 0 { + return; + } + let est_total = table.samples * interval as u64; + eprintln!( + "[alloc-site] {label}: interval={interval} samples={} est_bytes={est_total} inline_trips={} sites={}", + table.samples, + table.inline_trips, + table.sites.len() + ); + let mut types: Vec<(usize, u64)> = table + .by_type + .iter() + .enumerate() + .filter(|(_, &c)| c > 0) + .map(|(t, &c)| (t, c)) + .collect(); + types.sort_by_key(|&(_, c)| std::cmp::Reverse(c)); + let mut line = String::from("[alloc-site] by-type:"); + for (t, c) in types { + line.push_str(&format!( + " {}={}MB", + type_name(t), + c * interval as u64 / (1024 * 1024) + )); + } + eprintln!("{line}"); + let mut sites: Vec<(&[usize; DEPTH], &Site)> = table.sites.iter().collect(); + sites.sort_by_key(|(_, s)| std::cmp::Reverse(s.samples)); + for (key, s) in sites.iter().take(30) { + let n = key.iter().position(|&p| p == 0).unwrap_or(DEPTH); + let mut top_types: Vec<(usize, u32)> = s + .by_type + .iter() + .enumerate() + .filter(|(_, &c)| c > 0) + .map(|(t, &c)| (t, c)) + .collect(); + top_types.sort_by_key(|&(_, c)| std::cmp::Reverse(c)); + let types: Vec = top_types + .iter() + .take(3) + .map(|(t, c)| format!("{}:{}%", type_name(*t), *c as u64 * 100 / s.samples)) + .collect(); + eprintln!( + "[alloc-site] est_bytes={} samples={} mean_obj={} types={} site={}", + s.samples * interval as u64, + s.samples, + s.sampled_bytes / s.samples, + types.join(","), + crate::error::describe_chain(&key[..n], 5) + ); + } + }); +} diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index a99232f485..70c4a2979c 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -28,6 +28,7 @@ pub fn arena_alloc(size: usize, align: usize) -> *mut u8 { let offset = (*inline_ptr).offset; let arena = &mut *arena_ptr; let current = arena.current; + super::alloc_sample::note_inline_sync(arena.blocks[current].offset, offset); arena.blocks[current].offset = offset; } let ptr = crate::arena::arena_cell_alloc(arena_ptr, size, align); @@ -41,7 +42,7 @@ pub fn arena_alloc(size: usize, align: usize) -> *mut u8 { let inline = &mut *inline_ptr; inline.data = data; inline.offset = offset; - inline.size = block_size; + inline.size = super::alloc_sample::inline_limit(offset, block_size); } ptr } @@ -78,6 +79,7 @@ pub(crate) fn arena_alloc_gc_no_collect(size: usize, align: usize, obj_type: u8) use crate::gc::{GcHeader, GC_FLAG_ARENA, GC_HEADER_SIZE}; let total = gc_padded_total_size(size, align); + super::alloc_sample::note(total, obj_type); // Old-gen birth walks page lists and can reserve — outside the contract. if crate::gc::is_large_object_total_size_for_type(total, obj_type) { return std::ptr::null_mut(); @@ -123,6 +125,7 @@ fn arena_alloc_no_collect(size: usize, align: usize) -> *mut u8 { let offset = (*inline_ptr).offset; let arena = &mut *arena_ptr; let current = arena.current; + super::alloc_sample::note_inline_sync(arena.blocks[current].offset, offset); arena.blocks[current].offset = offset; } let Some(ptr) = crate::arena::arena_cell_try_alloc_current(arena_ptr, size, align) else { @@ -137,7 +140,7 @@ fn arena_alloc_no_collect(size: usize, align: usize) -> *mut u8 { let inline = &mut *inline_ptr; inline.data = data; inline.offset = offset; - inline.size = block_size; + inline.size = super::alloc_sample::inline_limit(offset, block_size); } ptr } @@ -170,6 +173,7 @@ pub fn arena_alloc_gc_longlived(size: usize, align: usize, obj_type: u8) -> *mut // assumes this invariant. let pad = align.max(8); let total = (GC_HEADER_SIZE + size + pad - 1) & !(pad - 1); + super::alloc_sample::note(total, obj_type); let raw = arena_alloc_longlived(total, align); unsafe { @@ -275,6 +279,7 @@ pub fn arena_alloc_gc_old(size: usize, align: usize, obj_type: u8) -> *mut u8 { pub(crate) fn arena_alloc_gc_old_born_tenured(size: usize, align: usize, obj_type: u8) -> *mut u8 { use crate::gc::{GcHeader, GC_FLAG_TENURED, GC_HEADER_SIZE}; + super::alloc_sample::note(gc_padded_total_size(size, align), obj_type); let user_ptr = arena_alloc_gc_old(size, align, obj_type); unsafe { let header = user_ptr.sub(GC_HEADER_SIZE) as *mut GcHeader; @@ -422,6 +427,7 @@ pub fn arena_alloc_gc(size: usize, align: usize, obj_type: u8) -> *mut u8 { // (`shapes.ts` sat 16 bytes over the flat 16 KB line and re-marked 118 006 // slots per minor because of it). let total = gc_padded_total_size(size, align); + super::alloc_sample::note(total, obj_type); if crate::gc::is_large_object_total_size_for_type(total, obj_type) { let user_ptr = arena_alloc_gc_old(size, align, obj_type); unsafe { diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index 0e8a09e487..6908be38da 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -623,7 +623,7 @@ impl Arena { let block = &self.blocks[self.current]; inline.data = block.data; inline.offset = block.offset; - inline.size = block.size; + inline.size = super::alloc_sample::inline_limit(block.offset, block.size); } }); } diff --git a/crates/perry-runtime/src/arena/inline.rs b/crates/perry-runtime/src/arena/inline.rs index 09071378c8..c186516788 100644 --- a/crates/perry-runtime/src/arena/inline.rs +++ b/crates/perry-runtime/src/arena/inline.rs @@ -43,7 +43,7 @@ pub extern "C" fn js_inline_arena_state() -> *mut InlineArenaState { let block = &arena.blocks[arena.current]; state.data = block.data; state.offset = block.offset; - state.size = block.size; + state.size = super::alloc_sample::inline_limit(block.offset, block.size); } state as *mut InlineArenaState } @@ -82,6 +82,7 @@ pub extern "C" fn js_inline_arena_slow_alloc( { let arena = &mut *arena_ptr; let current = arena.current; + super::alloc_sample::note_inline_sync(arena.blocks[current].offset, offset); arena.blocks[current].offset = offset; } // Allocate via existing path (may push a new block + run GC). @@ -95,7 +96,7 @@ pub extern "C" fn js_inline_arena_slow_alloc( let state_ref = &mut *state; state_ref.data = data; state_ref.offset = block_offset; - state_ref.size = block_size; + state_ref.size = super::alloc_sample::inline_limit(block_offset, block_size); ptr }) } @@ -113,7 +114,9 @@ pub fn sync_inline_arena_state() { if !state.data.is_null() { ARENA.with(|a| { let arena = &mut *(*a).get(); - arena.blocks[arena.current].offset = state.offset; + let current = arena.current; + super::alloc_sample::note_inline_sync(arena.blocks[current].offset, state.offset); + arena.blocks[current].offset = state.offset; }); } }); @@ -135,7 +138,9 @@ pub fn arena_start_fresh_general_block() { ARENA.with(|a| { let arena = &mut *(*a).get(); if !inline.data.is_null() { - arena.blocks[arena.current].offset = inline.offset; + let current = arena.current; + super::alloc_sample::note_inline_sync(arena.blocks[current].offset, inline.offset); + arena.blocks[current].offset = inline.offset; } if arena.blocks[arena.current].offset < FRESH_GENERAL_BLOCK_MIN_USED_BYTES { return; @@ -145,7 +150,7 @@ pub fn arena_start_fresh_general_block() { let block = &arena.blocks[arena.current]; inline.data = block.data; inline.offset = block.offset; - inline.size = block.size; + inline.size = super::alloc_sample::inline_limit(block.offset, block.size); } }); }); diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 9ba5401b19..59ffb4cac6 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -8,6 +8,7 @@ pub(crate) use std::alloc::{alloc, Layout}; pub(crate) use std::cell::{Cell, RefCell, UnsafeCell}; pub(crate) use std::collections::hash_map::Entry; +pub(crate) mod alloc_sample; mod allocators; mod block; mod inline; diff --git a/crates/perry-runtime/src/arena/promote.rs b/crates/perry-runtime/src/arena/promote.rs index 9976125ba9..12debf1b57 100644 --- a/crates/perry-runtime/src/arena/promote.rs +++ b/crates/perry-runtime/src/arena/promote.rs @@ -594,7 +594,7 @@ fn reset_young_after_promotion() { let block = &arena.blocks[arena.current]; inline.data = block.data; inline.offset = block.offset; - inline.size = block.size; + inline.size = super::alloc_sample::inline_limit(block.offset, block.size); } }); }); diff --git a/crates/perry-runtime/src/arena/quarantine.rs b/crates/perry-runtime/src/arena/quarantine.rs index 3db7280834..37986e7e06 100644 --- a/crates/perry-runtime/src/arena/quarantine.rs +++ b/crates/perry-runtime/src/arena/quarantine.rs @@ -574,7 +574,7 @@ pub(crate) fn copying_quarantine_from_spaces_and_flip() -> ArenaResetStats { let block = &arena.blocks[arena.current]; inline.data = block.data; inline.offset = block.offset; - inline.size = block.size; + inline.size = super::alloc_sample::inline_limit(block.offset, block.size); } }); }); diff --git a/crates/perry-runtime/src/arena/reset.rs b/crates/perry-runtime/src/arena/reset.rs index a5ba37fa86..e53918b18d 100644 --- a/crates/perry-runtime/src/arena/reset.rs +++ b/crates/perry-runtime/src/arena/reset.rs @@ -30,7 +30,7 @@ pub fn arena_reset_all_blocks_to_zero() { let block = &arena.blocks[0]; inline.data = block.data; inline.offset = 0; - inline.size = block.size; + inline.size = super::alloc_sample::inline_limit(0, block.size); } }); }); @@ -221,7 +221,7 @@ pub(crate) fn copying_reset_from_spaces_and_flip() -> ArenaResetStats { let block = &arena.blocks[arena.current]; inline.data = block.data; inline.offset = block.offset; - inline.size = block.size; + inline.size = super::alloc_sample::inline_limit(block.offset, block.size); } }); }); @@ -496,7 +496,7 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { if !block.data.is_null() { inline.data = block.data; inline.offset = block.offset; - inline.size = block.size; + inline.size = super::alloc_sample::inline_limit(block.offset, block.size); } } }); @@ -808,7 +808,8 @@ impl ArenaResetEmptyBlocksState { if !block.data.is_null() { inline.data = block.data; inline.offset = block.offset; - inline.size = block.size; + inline.size = + super::alloc_sample::inline_limit(block.offset, block.size); } } } diff --git a/crates/perry-runtime/src/builtins/formatting.rs b/crates/perry-runtime/src/builtins/formatting.rs index 3743f7dea7..a76ed50948 100644 --- a/crates/perry-runtime/src/builtins/formatting.rs +++ b/crates/perry-runtime/src/builtins/formatting.rs @@ -18,7 +18,7 @@ mod collection_equality; mod errors; pub(crate) use boxed_primitives::{ boxed_primitive_json_value, boxed_primitive_payload, boxed_primitive_to_string_tag, - prune_dead_boxed_primitive_payload_owners, + boxed_string_wrapper_utf16_len, prune_dead_boxed_primitive_payload_owners, }; pub use boxed_primitives::{ js_boxed_bigint_new, js_boxed_boolean_new, js_boxed_number_new, js_boxed_string_new, diff --git a/crates/perry-runtime/src/builtins/formatting/boxed_primitives.rs b/crates/perry-runtime/src/builtins/formatting/boxed_primitives.rs index 08c9a151c4..91cd9b4438 100644 --- a/crates/perry-runtime/src/builtins/formatting/boxed_primitives.rs +++ b/crates/perry-runtime/src/builtins/formatting/boxed_primitives.rs @@ -44,9 +44,8 @@ pub(super) unsafe fn boxed_primitive_base_for_object( /// String, otherwise `None`. /// /// The count is in UTF-16 code units, NOT Unicode scalar values: the index -/// properties are installed over `0..js_string_length` (`utf16_len`) by -/// `install_string_wrapper_indices`, so a non-BMP char (e.g. an emoji, two -/// UTF-16 units) occupies two indices. Counting `.chars()` would under-count +/// properties span `0..js_string_length` (`utf16_len`), so a non-BMP char +/// (e.g. an emoji, two UTF-16 units) occupies two indices. Counting `.chars()` would under-count /// and leak a trailing index (e.g. `new String("a😀b")` → `{ 3: 'b' }`). pub(super) unsafe fn boxed_string_char_index_count( obj_ptr: *const crate::object::ObjectHeader, @@ -139,6 +138,8 @@ fn attach_boxed_primitive_prototype(obj: *mut crate::object::ObjectHeader, class if obj.is_null() { return; } + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_h = scope.root_raw_mut_ptr(obj); let Some(name) = boxed_constructor_name(class_id) else { return; }; @@ -151,7 +152,12 @@ fn attach_boxed_primitive_prototype(obj: *mut crate::object::ObjectHeader, class let proto = crate::closure::closure_get_dynamic_prop(ctor_ptr, "prototype"); let proto_value = crate::value::JSValue::from_bits(proto.to_bits()); if proto_value.is_pointer() { - crate::object::prototype_chain::object_set_static_prototype(obj as usize, proto.to_bits()); + obj_h.with_mut_ptr(|obj: *mut crate::object::ObjectHeader| { + crate::object::prototype_chain::object_set_static_prototype( + obj as usize, + proto.to_bits(), + ) + }); } } @@ -162,45 +168,40 @@ fn install_string_wrapper_length( if obj.is_null() || string_ptr.is_null() { return; } - let key = crate::string::js_string_from_bytes(b"length".as_ptr(), 6); let len = crate::string::js_string_length(string_ptr) as f64; - crate::object::js_object_set_field_by_name(obj, key, len); - crate::object::set_builtin_property_attrs( - obj as usize, - "length".to_string(), - crate::object::PropertyAttrs::new(false, false, false), - ); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_h = scope.root_raw_mut_ptr(obj); + let key = crate::string::js_string_from_bytes(b"length".as_ptr(), 6); + obj_h.with_mut_ptr(|obj| crate::object::js_object_set_field_by_name(obj, key, len)); + obj_h.with_mut_ptr(|obj: *mut crate::object::ObjectHeader| { + crate::object::set_builtin_property_attrs( + obj as usize, + "length".to_string(), + crate::object::PropertyAttrs::new(false, false, false), + ) + }); } -/// String exotic objects (ECMA-262 §10.4.3) expose each UTF-16 code unit as an -/// integer-indexed own property `"0".."len-1"` with the descriptor -/// `{ value: , writable: false, enumerable: true, configurable: false }`. -/// `new String("abc")` therefore reports `getOwnPropertyDescriptor(s, "0")`, -/// `s.hasOwnProperty("0")`, and `Object.keys(s)`/enumeration over the indices. -/// Installed eagerly at construction (typical `new String` receivers are -/// short); the wrapper's `length` is installed separately and stays last. -fn install_string_wrapper_indices( - obj: *mut crate::object::ObjectHeader, - string_ptr: *const crate::string::StringHeader, -) { - if obj.is_null() || string_ptr.is_null() { - return; - } - let len = crate::string::js_string_length(string_ptr); - for i in 0..len { - let ch = crate::string::js_string_char_at(string_ptr, i as i32); - if ch.is_null() { - continue; +/// UTF-16 length of the primitive a `String` wrapper boxes, or `None` when +/// `addr` is not one. Two header reads and a side-table probe: no content +/// copy, no allocation, so a descriptor lookup can afford to ask. +pub(crate) fn boxed_string_wrapper_utf16_len(addr: usize) -> Option { + unsafe { + let header = crate::value::addr_class::try_read_gc_header(addr)?; + if header.obj_type != crate::gc::GC_TYPE_OBJECT { + return None; } - let name = i.to_string(); - let key = crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32); - let ch_value = f64::from_bits(crate::value::JSValue::string_ptr(ch).bits()); - crate::object::js_object_set_field_by_name(obj, key, ch_value); - crate::object::set_builtin_property_attrs( - obj as usize, - name, - crate::object::PropertyAttrs::new(false, true, false), - ); + let obj_ptr = addr as *const crate::object::ObjectHeader; + let (class_id, payload) = boxed_primitive_payload_for_object(obj_ptr)?; + if class_id != CLASS_ID_BOXED_STRING { + return None; + } + let str_ptr = crate::value::js_get_string_pointer_unified(payload) + as *const crate::string::StringHeader; + if str_ptr.is_null() { + return None; + } + Some(crate::string::js_string_length(str_ptr)) } } @@ -327,8 +328,8 @@ pub extern "C" fn js_boxed_string_new(value: f64, has_arg: i32) -> f64 { // empty-string case and `js_string_coerce` otherwise, the latter running a // user `toString`/`valueOf` for a POINTER_TAG value — so either can collect // and EVACUATE while `obj` sits in a raw Rust local. Every use below - // (`register_boxed_primitive_payload`, the two `install_string_wrapper_*` - // calls, `attach_boxed_primitive_prototype`, and the returned NaN-box) + // (`register_boxed_primitive_payload`, the `install_string_wrapper_length` + // call, `attach_boxed_primitive_prototype`, and the returned NaN-box) // dereferences or keys on it. let scope = crate::gc::RuntimeHandleScope::new(); let obj_handle = scope.root_raw_mut_ptr(obj); @@ -348,10 +349,13 @@ pub extern "C" fn js_boxed_string_new(value: f64, has_arg: i32) -> f64 { }); let boxed = f64::from_bits(crate::value::JSValue::string_ptr(ptr).bits()); register_boxed_primitive_payload(obj, boxed); - install_string_wrapper_indices(obj, ptr); + // #9810: character indices are virtual String exotic properties. Index + // keys and values are produced on demand; boxing never walks the string. install_string_wrapper_length(obj, ptr); - attach_boxed_primitive_prototype(obj, CLASS_ID_BOXED_STRING); - crate::value::js_nanbox_pointer(obj as i64) + obj_handle.with_mut_ptr(|obj| attach_boxed_primitive_prototype(obj, CLASS_ID_BOXED_STRING)); + obj_handle.with_mut_ptr(|obj: *mut crate::object::ObjectHeader| { + crate::value::js_nanbox_pointer(obj as i64) + }) } #[no_mangle] diff --git a/crates/perry-runtime/src/builtins/mod.rs b/crates/perry-runtime/src/builtins/mod.rs index 3d21c1c681..66c36794e1 100644 --- a/crates/perry-runtime/src/builtins/mod.rs +++ b/crates/perry-runtime/src/builtins/mod.rs @@ -163,10 +163,11 @@ pub use formatting::{ pub(crate) use formatting::{ boxed_primitive_json_value, boxed_primitive_payload, boxed_primitive_to_string_tag, - format_finite_number_js, format_jsvalue, int32_or_class_repr, is_array_hole, is_negative_zero, - jsvalue_string_content, prune_dead_boxed_primitive_payload_owners, InspectCompactGuard, - InspectCustomInspectGuard, InspectDepthLimitGuard, InspectGettersGuard, InspectShowHiddenGuard, - InspectSortedGuard, INT_EXACT_FASTPATH_LIMIT, + boxed_string_wrapper_utf16_len, format_finite_number_js, format_jsvalue, int32_or_class_repr, + is_array_hole, is_negative_zero, jsvalue_string_content, + prune_dead_boxed_primitive_payload_owners, InspectCompactGuard, InspectCustomInspectGuard, + InspectDepthLimitGuard, InspectGettersGuard, InspectShowHiddenGuard, InspectSortedGuard, + INT_EXACT_FASTPATH_LIMIT, }; #[cfg(test)] pub(crate) use formatting::{ diff --git a/crates/perry-runtime/src/error.rs b/crates/perry-runtime/src/error.rs index e6a55d645f..8542c6b3dd 100644 --- a/crates/perry-runtime/src/error.rs +++ b/crates/perry-runtime/src/error.rs @@ -1936,7 +1936,8 @@ static KEEP_ERROR_IS_ERROR: extern "C" fn(f64) -> f64 = js_error_is_error; #[path = "error_stack_frames.rs"] mod stack_frames; pub(crate) use stack_frames::{ - capture_frames_payload, frames_payload_to_lines, materialize_error_stack, + capture_frames_payload, capture_ips, describe_chain, frames_payload_to_lines, + materialize_error_stack, MAX_CAPTURED_FRAMES, }; #[path = "error_subclass_stack.rs"] diff --git a/crates/perry-runtime/src/error_stack_frames.rs b/crates/perry-runtime/src/error_stack_frames.rs index 1f062cb14e..ba70ccc5f5 100644 --- a/crates/perry-runtime/src/error_stack_frames.rs +++ b/crates/perry-runtime/src/error_stack_frames.rs @@ -321,6 +321,60 @@ pub(crate) fn capture_encoded() -> ([u8; MAX_CAPTURED_FRAMES * PC_CHARS], usize) (blob, len) } +/// Raw return addresses of the current native stack, innermost first, for +/// the GC's site-attribution diagnostics (`gc/diag_sites.rs`, the +/// `PERRY_ALLOC_SITE_SAMPLE` sampler). Same walk as `capture_encoded`, no +/// encoding. +pub(crate) fn capture_ips(out: &mut [usize; MAX_CAPTURED_FRAMES]) -> usize { + walk::capture(out) +} + +/// Best-effort one-line description of a code address for diagnostics: the +/// registered JS display name when `ip` is inside a compiled user function, +/// else the nearest linker symbol (`dladdr`), else the bare address. Never +/// called on a hot path — the JS-name index takes a lock and may rebuild. +pub(crate) fn describe_ip(ip: usize) -> String { + let js = with_index(|index| { + name_for_ip(index, ip.saturating_sub(1)) + .and_then(|n| std::str::from_utf8(n).ok().map(|s| s.to_string())) + }) + .flatten(); + if let Some(name) = js.filter(|n| !n.is_empty()) { + return format!("js:{name}"); + } + #[cfg(unix)] + { + let mut info: libc::Dl_info = unsafe { std::mem::zeroed() }; + // SAFETY: `dladdr` only reads the address and fills `info`. + if unsafe { libc::dladdr(ip as *const libc::c_void, &mut info) } != 0 + && !info.dli_sname.is_null() + { + let name = unsafe { std::ffi::CStr::from_ptr(info.dli_sname) }.to_string_lossy(); + let off = ip.saturating_sub(info.dli_saddr as usize); + let mut n = name.into_owned(); + if n.len() > 72 { + n.truncate(72); + } + return format!("{n}+{off:#x}"); + } + } + format!("{ip:#x}") +} + +/// `describe_ip` for a chain, innermost first, skipping frames inside `skip` +/// (a set of symbol-name substrings the caller considers plumbing). Returns +/// up to `max` descriptions joined by ` < `. +pub(crate) fn describe_chain(pcs: &[usize], max: usize) -> String { + let mut out = Vec::with_capacity(max); + for &pc in pcs { + if out.len() >= max { + break; + } + out.push(describe_ip(pc)); + } + out.join(" < ") +} + // --------------------------------------------------------------------------- // Resolution: address -> JS display name. // --------------------------------------------------------------------------- diff --git a/crates/perry-runtime/src/gc/census.rs b/crates/perry-runtime/src/gc/census.rs index 99827e8720..743c93fb69 100644 --- a/crates/perry-runtime/src/gc/census.rs +++ b/crates/perry-runtime/src/gc/census.rs @@ -50,7 +50,7 @@ crate::perry_thread_local! { } #[cfg(test)] -thread_local! { +crate::perry_thread_local! { /// Test-only per-thread override of the output path, so a unit test can /// enable the census without touching the process env (the env read is a /// process-wide OnceLock that any earlier collection would latch). diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index be5dfdab47..9724feb04c 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -170,6 +170,8 @@ pub(super) struct CopyingNurseryCollector { /// skipped. `debug_assert_no_remembering_possible` re-derives the premise at /// runtime in debug builds. pub(super) skip_remembering: bool, + /// `PERRY_GC_DIAG=1`: per-minor survival attribution (gc/survival_diag.rs). + pub(super) survival: Option>, /// Weak target slots (WeakRef referent / WeakMap-WeakSet entry key / /// FinalizationRegistry record target) seen during the copy scan. The /// scan must NOT evacuate through them (that would strengthen the weak @@ -242,12 +244,22 @@ impl CopyingNurseryCollector { live_from_bytes: 0, tenuring_survivals, skip_remembering: false, + survival: crate::gc::gc_diag_enabled() + .then(|| Box::new(super::survival_diag::SurvivalDiag::new())), weak_slots: Vec::new(), memo_addr: 0, memo_result: 0, } } + /// Mirror a `worklist.push` into the survival diag's origin vector. + #[inline] + fn survival_push(&mut self) { + if let Some(d) = self.survival.as_mut() { + d.note_worklist_push(); + } + } + pub(super) unsafe fn record_large_excluded(&mut self, header: *mut GcHeader) { if header.is_null() { return; @@ -390,6 +402,7 @@ impl CopyingNurseryCollector { if flags & (GC_FLAG_MARKED | GC_FLAG_PINNED) == 0 { (*ptr.header).gc_flags = flags | GC_FLAG_MARKED; self.worklist.push(ptr.header); + self.survival_push(); self.marked_headers.push(ptr.header); } } @@ -432,6 +445,10 @@ impl CopyingNurseryCollector { (*header).gc_flags = flags | GC_FLAG_MARKED; let total = (*header).size as usize; self.worklist.push(header); + self.survival_push(); + if let Some(d) = self.survival.as_mut() { + d.record((*header).obj_type, total, true); + } self.moved_headers.push(header); self.stats.promoted_objects += 1; self.stats.promoted_bytes += total; @@ -550,6 +567,10 @@ impl CopyingNurseryCollector { gc_type_after_payload_move((*header).obj_type, old_user as usize, new_user as usize); self.worklist.push(new_header); + self.survival_push(); + if let Some(d) = self.survival.as_mut() { + d.record((*new_header).obj_type, total, promote); + } self.moved_headers.push(new_header); self.live_from_bytes += total; if promote { @@ -633,11 +654,17 @@ impl CopyingNurseryCollector { } let header = self.worklist[i]; i += 1; + if let Some(d) = self.survival.as_mut() { + d.begin_drain_entry(i - 1); + } if (*header).gc_flags & GC_FLAG_FORWARDED != 0 { continue; } self.scan_object_fields(header); } + if let Some(d) = self.survival.as_mut() { + d.end_drain(); + } } /// Second pass over the weak target slots collected during the scan: @@ -1363,6 +1390,9 @@ pub(super) fn run_copied_minor_attempt( &snapshot, Some(&mut dirty_scan_covered), |slot, header, external, stats| unsafe { + if let Some(d) = collector.survival.as_mut() { + d.remembered_parent_type = (*header).obj_type; + } let before = *slot; collector.visit_slot_with_parent(slot, header, external); if *slot != before { @@ -1803,6 +1833,11 @@ pub(super) fn run_copied_minor_attempt( super::policy::GC_AT_DECLARED_SAFEPOINT.with(std::cell::Cell::get) ); } + if let Some(d) = collector.survival.as_ref() { + d.report(super::survival_diag::next_minor_seq()); + } + crate::arena::alloc_sample::report("minor"); + super::diag_sites::report_primitive_dispatch("minor"); report_forwarding_refusals("copying_minor"); super::scanner_profile::report_and_reset("copying_minor"); CopiedMinorAttempt::Done(Some(CopiedMinorFastPathOutcome { diff --git a/crates/perry-runtime/src/gc/diag_sites.rs b/crates/perry-runtime/src/gc/diag_sites.rs new file mode 100644 index 0000000000..4044afb4e0 --- /dev/null +++ b/crates/perry-runtime/src/gc/diag_sites.rs @@ -0,0 +1,428 @@ +//! `PERRY_GC_DIAG=1`: WHY a collection was decided, WHICH arm ran a full +//! mark-sweep, and WHAT the budgeted collector charged the mutator per cycle +//! and per charge site. +//! +//! The per-cycle diag lines (`[gc-copy-minor]`, `[gc-step]`, `[gc]`) report +//! what a collection did; none of them says why it was scheduled. On the +//! compiled claude-code TUI a 400-character streamed reply cost 42 copying +//! minors — most of them over a nearly empty Eden — plus ten back-to-back +//! synchronous full mark-sweeps from the allocation-point old-reclaim arm, +//! and the only way to tell which predicate fired was to re-derive every input +//! by hand. These lines print the inputs at the decision: +//! +//! * `[gc-trigger] site=… kind=…` — every predicate input the trigger policy +//! reads (`arena_total` vs the armed base trigger, from-space occupancy vs +//! the nursery cap, old-gen reclaimable pressure vs its baseline and band, +//! the malloc-count pair, the pending/retaining flags), emitted at each +//! site that decides to collect. +//! * `[gc-full] site=… trigger=…` — one line per full mark-sweep, naming the +//! arm that started it (`alloc_point_old_reclaim`, `safepoint_old_reclaim`, +//! `budgeted`, `manual`, …) with a running per-site count. +//! * `[gc-budgeted] start|done …` — one pair per budgeted (incremental) cycle: +//! trigger, full/minor, how many steps drove it, the wall time of those +//! steps split by cycle phase, and the root-scan share of the total. +//! * `[gc-charge] …` — the mutator-assist and synchronous-full work charged +//! to each calling site (return-address chain, resolved to the JS display +//! name where the frame is compiled user code), so "which JS operation is +//! paying for the collector" is a counter rather than a profile guess. +//! +//! Everything here is gated on [`gc_diag_enabled`] and costs one cached-bool +//! read when off. + +use super::*; +use std::collections::HashMap; +use std::time::Instant; + +/// Print the predicate inputs behind a collection decision. +pub(super) fn trigger_decision(site: &'static str, kind: &'static str) { + if !gc_diag_enabled() { + return; + } + let arena_total = crate::arena::arena_total_bytes(); + let next_base = policy::next_arena_trigger_base(); + let armed = policy::GC_TRIGGER_ARMED.with(Cell::get); + let from_space = crate::arena::copying_from_space_in_use_bytes(); + let nursery_cap = tenuring::scavenge_nursery_cap_effective_bytes(); + let old_reclaimable = policy::old_gen_reclaimable_pressure_bytes(); + let external = policy::external_side_live_bytes(); + let old_baseline = policy::GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(Cell::get); + let old_band = policy::gc_old_reclaim_growth_band_bytes(old_baseline); + let old_threshold = gc_old_gen_reclaim_threshold_dyn_bytes(); + let old_pending = policy::GC_OLD_RECLAIM_PENDING.with(Cell::get); + let retaining = policy::GC_MAJOR_PACING_RETAINING.with(Cell::get); + let malloc = malloc_object_count(); + let next_malloc = policy::GC_NEXT_MALLOC_TRIGGER.with(Cell::get); + let old_in_use = crate::arena::old_gen_in_use_bytes(); + let old_free = old_free_bytes(); + eprintln!( + "[gc-trigger] site={site} kind={kind} arena_total={arena_total} next_base={next_base} armed={armed} \ + from_space={from_space} nursery_cap={nursery_cap} old_in_use={old_in_use} old_free={old_free} \ + old_reclaimable={old_reclaimable} external_side={external} old_baseline={old_baseline} \ + old_band={old_band} old_threshold={old_threshold} old_pending={old_pending} retaining={retaining} \ + malloc={malloc} next_malloc={next_malloc}" + ); +} + +crate::perry_thread_local! { + /// Label the arm that is about to run a synchronous full leaves for the + /// chokepoint (`gc_collect_full_mark_sweep_with_trigger`) to consume. + static FULL_SITE: Cell> = const { Cell::new(None) }; +} + +/// Name the arm behind the next synchronous full mark-sweep. +pub(super) fn set_full_site(site: &'static str) { + FULL_SITE.with(|s| s.set(Some(site))); +} + +/// Consume the pending arm label; `sync` when none was set (manual `gc()`, +/// emergency, escalation). +pub(super) fn take_full_site() -> &'static str { + FULL_SITE.with(|s| s.take()).unwrap_or("sync") +} + +crate::perry_thread_local! { + static FULL_SITE_COUNTS: RefCell> = const { RefCell::new(Vec::new()) }; + static BUDGETED: RefCell> = const { RefCell::new(None) }; + static CHARGES: RefCell> = RefCell::new(HashMap::new()); +} + +/// Test-only: how many synchronous fulls `full_started` counted at `site`. +#[cfg(test)] +pub(super) fn test_full_site_count(site: &str) -> u32 { + FULL_SITE_COUNTS.with(|c| { + c.borrow() + .iter() + .find(|(s, _)| *s == site) + .map_or(0, |(_, n)| *n) + }) +} + +/// Test-only: `(calls, units, us, fulls, minors)` of every charge row. +#[cfg(test)] +pub(super) fn test_charge_rows() -> Vec<(u64, u64, u64, u64, u64)> { + CHARGES.with(|c| { + c.borrow() + .values() + .map(|r| (r.calls, r.units, r.us, r.fulls, r.minors)) + .collect() + }) +} + +/// Test-only: `(steps, step_us, units, root_scan_us)` of the last completed +/// budgeted cycle's accounting. +#[cfg(test)] +pub(super) fn test_last_budgeted() -> Option<(u64, u64, u64, u64)> { + LAST_BUDGETED.with(Cell::get) +} + +#[cfg(test)] +crate::perry_thread_local! { + static LAST_BUDGETED: Cell> = const { Cell::new(None) }; +} + +/// One full mark-sweep is starting from `site`. +pub(super) fn full_started(site: &'static str, trigger: GcTriggerKind) { + if !gc_diag_enabled() { + return; + } + let count = FULL_SITE_COUNTS.with(|c| { + let mut c = c.borrow_mut(); + if let Some(entry) = c.iter_mut().find(|(s, _)| *s == site) { + entry.1 += 1; + entry.1 + } else { + c.push((site, 1)); + 1 + } + }); + eprintln!( + "[gc-full] site={site} trigger={trigger:?} count_at_site={count} old_reclaimable={} old_baseline={}", + policy::old_gen_reclaimable_pressure_bytes(), + policy::GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(Cell::get) + ); +} + +/// `GcCyclePhase::ffi_code()` runs 1..=8; index by it directly. +const PHASE_SLOTS: usize = 9; + +struct BudgetedCycleDiag { + trigger: GcTriggerKind, + collection: &'static str, + progress: &'static str, + started: Instant, + steps: u64, + step_us: u64, + units: u64, + phase_us: [u64; PHASE_SLOTS], + phase_steps: [u64; PHASE_SLOTS], +} + +/// A budgeted cycle was just installed as the active cycle. +pub(super) fn budgeted_started( + trigger: GcTriggerKind, + collection: GcCollectionKind, + progress: GcProgressKind, +) { + if !gc_diag_enabled() { + return; + } + let collection = match collection { + GcCollectionKind::Full => "full", + GcCollectionKind::Minor => "minor", + }; + eprintln!( + "[gc-budgeted] start trigger={trigger:?} kind={collection} progress={} old_reclaimable={} old_baseline={} arena_total={}", + progress.as_str(), + policy::old_gen_reclaimable_pressure_bytes(), + policy::GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(Cell::get), + crate::arena::arena_total_bytes() + ); + BUDGETED.with(|b| { + *b.borrow_mut() = Some(BudgetedCycleDiag { + trigger, + collection, + progress: progress.as_str(), + started: Instant::now(), + steps: 0, + step_us: 0, + units: 0, + phase_us: [0; PHASE_SLOTS], + phase_steps: [0; PHASE_SLOTS], + }); + }); +} + +/// One budgeted step ran: `phase_before` is the phase it started in. +pub(super) fn budgeted_step_done(phase_code: u32, elapsed_us: u64, units: usize) { + if !gc_diag_enabled() { + return; + } + BUDGETED.with(|b| { + if let Some(d) = b.borrow_mut().as_mut() { + d.steps += 1; + d.step_us += elapsed_us; + d.units = d.units.saturating_add(units as u64); + let slot = (phase_code as usize).min(PHASE_SLOTS - 1); + d.phase_us[slot] += elapsed_us; + d.phase_steps[slot] += 1; + } + }); +} + +/// The active budgeted cycle completed and was rebaselined. +pub(super) fn budgeted_completed(freed_bytes: u64) { + if !gc_diag_enabled() { + return; + } + let Some(d) = BUDGETED.with(|b| b.borrow_mut().take()) else { + return; + }; + #[cfg(test)] + LAST_BUDGETED.with(|c| c.set(Some((d.steps, d.step_us, d.units, d.phase_us[2])))); + const NAMES: [&str; PHASE_SLOTS] = [ + "?", + "build_valid_ptrs", + "root_scan", + "mark", + "block_persist", + "atomic_finalize", + "sweep", + "reclaim", + "complete", + ]; + let mut phases = String::new(); + for (name, (us, steps)) in NAMES + .iter() + .zip(d.phase_us.iter().zip(d.phase_steps.iter())) + .skip(1) + { + if *steps == 0 { + continue; + } + phases.push_str(&format!(" {name}={us}us/{steps}steps")); + } + let root_share = (d.phase_us[2] * 1000).checked_div(d.step_us).unwrap_or(0); + eprintln!( + "[gc-budgeted] done trigger={:?} kind={} progress={} steps={} step_us={} units={} wall_us={} freed={} root_scan_permille={} phases:{}", + d.trigger, + d.collection, + d.progress, + d.steps, + d.step_us, + d.units, + d.started.elapsed().as_micros(), + freed_bytes, + root_share, + phases + ); + report_charges("budgeted-done"); +} + +/// Return-address chain depth kept per charge site. Frame 0 is the caller of +/// `gc_check_trigger` (the allocator or `js_json_parse`); the next ones walk +/// out to the compiled JS function that issued the allocation. +const CHARGE_DEPTH: usize = 6; + +#[derive(Default, Clone, Copy)] +struct Charge { + calls: u64, + units: u64, + us: u64, + minors: u64, + fulls: u64, +} + +/// Wraps one `gc_check_trigger` arm: captures the caller chain on `begin` +/// (only under the diag), and on `end` charges the elapsed time and the work +/// units it drove to that chain. +pub(super) struct ChargeProbe { + pcs: [usize; crate::error::MAX_CAPTURED_FRAMES], + n: usize, + started: Option, +} + +impl ChargeProbe { + #[inline] + pub(super) fn begin() -> Self { + let mut probe = Self { + pcs: [0; crate::error::MAX_CAPTURED_FRAMES], + n: 0, + started: None, + }; + if gc_diag_enabled() { + probe.n = crate::error::capture_ips(&mut probe.pcs); + probe.started = Some(Instant::now()); + } + probe + } + + /// `kind`: what the arm did — `assist` (budgeted step), `sync_full`, + /// `direct_minor`. + pub(super) fn end(self, units: usize, kind: ChargeKind) { + let Some(started) = self.started else { + return; + }; + let us = started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + let mut key = [0usize; CHARGE_DEPTH]; + // Skip frame 0: it is the return into `gc_check_trigger` itself. + let chain = &self.pcs[1.min(self.n)..self.n]; + for (slot, pc) in key.iter_mut().zip(chain) { + *slot = *pc; + } + CHARGES.with(|c| { + let mut c = c.borrow_mut(); + let e = c.entry(key).or_default(); + e.calls += 1; + e.units = e.units.saturating_add(units as u64); + e.us += us; + match kind { + ChargeKind::Assist => {} + ChargeKind::SyncFull => e.fulls += 1, + ChargeKind::DirectMinor => e.minors += 1, + } + }); + } +} + +#[derive(Clone, Copy)] +pub(super) enum ChargeKind { + Assist, + SyncFull, + DirectMinor, +} + +/// Print the heaviest charge sites since the last report, then reset. +pub(super) fn report_charges(label: &str) { + if !gc_diag_enabled() { + return; + } + let rows: Vec<([usize; CHARGE_DEPTH], Charge)> = + CHARGES.with(|c| c.borrow_mut().drain().collect()); + if rows.is_empty() { + return; + } + let total_us: u64 = rows.iter().map(|(_, r)| r.us).sum(); + let total_calls: u64 = rows.iter().map(|(_, r)| r.calls).sum(); + let mut rows = rows; + rows.sort_by_key(|(_, r)| std::cmp::Reverse(r.us)); + eprintln!( + "[gc-charge] {label}: sites={} calls={total_calls} total_us={total_us}", + rows.len() + ); + for (key, r) in rows.iter().take(12) { + let n = key.iter().position(|&p| p == 0).unwrap_or(CHARGE_DEPTH); + eprintln!( + "[gc-charge] us={} calls={} units={} fulls={} minors={} site={}", + r.us, + r.calls, + r.units, + r.fulls, + r.minors, + crate::error::describe_chain(&key[..n], 5) + ); + } +} + +// --- primitive-method dispatch tower ------------------------------------- +// +// A method call whose receiver is a string/number/boolean/bigint primitive and +// whose method the native dispatch tower does not recognise falls through to +// `native_call_method::call_primitive_builtin_prototype_method`: it resolves +// `globalThis..prototype[]` and, for a SLOPPY callee, boxes +// the receiver with `ToObject`. For a string that wrapper materialises one own +// property per UTF-16 code unit. So a single unrecognised method name on a hot +// render path turns into O(length) allocations per call, and the only way to +// tell WHICH names those are is to count them at the fork. + +thread_local! { + /// `".prototype." -> (calls, receiver_utf16_chars)`. + static PRIMITIVE_DISPATCH: RefCell> = + RefCell::new(HashMap::new()); + /// String wrappers actually materialised: (wrappers, index properties). + static STRING_WRAPPERS: Cell<(u64, u64)> = const { Cell::new((0, 0)) }; +} + +/// Record one trip through the primitive-method fallback. `recv_chars` is the +/// receiver's UTF-16 length (0 when the receiver is not a string) — the number +/// of own index properties a sloppy callee's `ToObject` wrapper costs. +pub(crate) fn primitive_dispatch(builtin: &[u8], method: &str, recv_chars: u64) { + if !gc_diag_enabled() { + return; + } + let name = format!("{}.prototype.{method}", String::from_utf8_lossy(builtin)); + PRIMITIVE_DISPATCH.with(|m| { + let mut m = m.borrow_mut(); + let entry = m.entry(name).or_insert((0, 0)); + entry.0 += 1; + entry.1 += recv_chars; + }); +} + +/// Print the fallback histogram, hottest first. +pub(super) fn report_primitive_dispatch(label: &str) { + if !gc_diag_enabled() { + return; + } + let (wrappers, indices) = STRING_WRAPPERS.with(Cell::get); + if wrappers > 0 { + eprintln!( + "[gc-primitive-dispatch] {label}: string_wrappers={wrappers} index_properties={indices}" + ); + } + let rows: Vec<(String, (u64, u64))> = + PRIMITIVE_DISPATCH.with(|m| m.borrow().iter().map(|(k, v)| (k.clone(), *v)).collect()); + if rows.is_empty() { + return; + } + let calls: u64 = rows.iter().map(|(_, v)| v.0).sum(); + let chars: u64 = rows.iter().map(|(_, v)| v.1).sum(); + let mut rows = rows; + rows.sort_by_key(|(_, v)| std::cmp::Reverse(v.0)); + eprintln!( + "[gc-primitive-dispatch] {label}: names={} calls={calls} receiver_chars={chars}", + rows.len() + ); + for (name, (n, ch)) in rows.iter().take(20) { + eprintln!("[gc-primitive-dispatch] calls={n} receiver_chars={ch} {name}"); + } +} diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index cb0acd9a6b..844be2e5e9 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -158,11 +158,14 @@ mod prefetch; mod copying; mod copying_first_cycle; mod copying_pointer_set; +mod diag_sites; +pub(crate) use diag_sites::primitive_dispatch as diag_primitive_dispatch; /// #8174: shared validation for the TARGET of a forwarding pointer. mod forwarding; /// Per-scanner root attribution for the copied-minor root scan (#7915). mod scanner_profile; mod sticky_remembered; +mod survival_diag; /// #9754: per-side-table young-entry logs (remembered sets for the runtime /// side tables), so a minor-scoped root scan visits only the entries that /// can hold a pointer a minor acts on. @@ -798,6 +801,7 @@ fn gc_collect_full_mark_sweep_with_trigger(trigger: GcTriggerSnapshot) -> GcColl let _contract_heal = policy::contract_scan_heal_guard(); gc_drain_active_budgeted_cycle(); GC_TRIGGER_BUMPED.with(|c| c.set(false)); + diag_sites::full_started(diag_sites::take_full_site(), trigger.kind); GcCycleState::new_full(trigger).run_to_completion() } @@ -947,6 +951,7 @@ pub fn gc_init() { census::census_on_gc_init(); #[cfg(feature = "alloc-census")] crate::alloc_census::alloc_census_init(); + crate::arena::alloc_sample::init_from_env(); reg_budgeted_scanner!( scan_runtime_handle_roots_mut, scan_runtime_handle_roots_mut_step, @@ -1335,6 +1340,9 @@ pub extern "C" fn js_gc_release_current_thread_collection_side_allocations() { // once-only when the mode is off. schedule::report_exit_summary(); crate::r#box::report_box_stats_at_exit(); + crate::arena::alloc_sample::report("exit"); + diag_sites::report_charges("exit"); + diag_sites::report_primitive_dispatch("exit"); emit_incremental_liveness_diag(); emit_schedule_liveness_verdict(); } diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 0977d5e70a..7088e39b65 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -2350,6 +2350,9 @@ pub fn gc_check_trigger() { { let _reentry = OldReclaimReentryGuard::enter(); GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false)); + super::diag_sites::trigger_decision("alloc_point", "OldReclaim"); + super::diag_sites::set_full_site("alloc_point_old_reclaim"); + let probe = super::diag_sites::ChargeProbe::begin(); let _scan = super::roots::ManualGcScanGuard::force_full_scan( super::ConservativeScanSite::OldReclaimAllocPoint, ); @@ -2357,6 +2360,7 @@ pub fn gc_check_trigger() { GcTriggerKind::OldGenBytes, )) .emit_after_current(); + probe.end(0, super::diag_sites::ChargeKind::SyncFull); return; } @@ -2476,6 +2480,8 @@ pub fn gc_check_trigger() { } let pre_in_use = crate::arena::arena_in_use_bytes(); let pre_malloc_count = malloc_object_count(); + super::diag_sites::trigger_decision("alloc_point_slack", "nursery"); + let probe = super::diag_sites::ChargeProbe::begin(); // THE ALLOC POINT IS REGISTER-IMPRECISE, SO THIS MINOR MUST NOT // MOVE. Unconditional, and the unconditionality is the fix for // #7682. @@ -2557,6 +2563,7 @@ pub fn gc_check_trigger() { gc_finish_arena_trigger_collection(pre_in_use, outcome); } } + probe.end(0, super::diag_sites::ChargeKind::DirectMinor); return; } } @@ -2565,10 +2572,11 @@ pub fn gc_check_trigger() { return; } - let _ = gc_mutator_assist_step_work_units_inner_with_progress( - gc_mutator_assist_scaled_work_units(), - GcProgressKind::MutatorAssist, - ); + let units = gc_mutator_assist_scaled_work_units(); + let probe = super::diag_sites::ChargeProbe::begin(); + let _ = + gc_mutator_assist_step_work_units_inner_with_progress(units, GcProgressKind::MutatorAssist); + probe.end(units, super::diag_sites::ChargeKind::Assist); } /// Debt-proportional assist pacing (#6180 Stage 2, measured 2026-07-10). @@ -2795,6 +2803,8 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { } let _reentry = OldReclaimReentryGuard::enter(); GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false)); + super::diag_sites::trigger_decision("safepoint", "OldReclaim"); + super::diag_sites::set_full_site("safepoint_old_reclaim"); // No `force_full_scan`: roots are precise at this safepoint. gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture( GcTriggerKind::OldGenBytes, @@ -2821,6 +2831,13 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { }; let pre_in_use = crate::arena::arena_in_use_bytes(); let pre_malloc_count = malloc_object_count(); + super::diag_sites::trigger_decision( + "safepoint", + match kind { + GcTriggerKind::MallocCount => "MallocCount", + _ => "ArenaBytes", + }, + ); // No `force_full_scan`: roots are precise at this safepoint. let outcome = super::gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(kind)); match kind { @@ -3348,6 +3365,7 @@ fn gc_finish_budgeted_cycle(mut cycle: BudgetedGcCycle) -> JsGcStepResult { .state .take_outcome() .expect("completed budgeted GC cycle must produce an outcome"); + let freed_for_diag = outcome.freed_bytes; match cycle.rebaseline { BudgetedGcRebaseline::ArenaBytes { pre_in_use } => { gc_finish_arena_trigger_collection(pre_in_use, outcome); @@ -3363,6 +3381,7 @@ fn gc_finish_budgeted_cycle(mut cycle: BudgetedGcCycle) -> JsGcStepResult { } } GC_BUDGETED_CYCLE_ACTIVE.with(|active| active.set(false)); + super::diag_sites::budgeted_completed(freed_for_diag); gc_step_result( JS_GC_STEP_STATUS_COMPLETED, GcCyclePhase::Complete.ffi_code(), @@ -3543,8 +3562,14 @@ fn gc_budgeted_step_work_units_inner_with_progress( ); return gc_budgeted_skipped_result(); } + super::diag_sites::trigger_decision("budgeted_start", "due"); let cycle = gc_start_budgeted_cycle_for_pressure(start_progress_kind) .expect("budgeted GC pressure was observed before starting cycle"); + super::diag_sites::budgeted_started( + cycle.trigger_kind, + cycle.collection_kind, + start_progress_kind, + ); GC_BUDGETED_CYCLE.with(|slot| { *slot.borrow_mut() = Some(cycle); }); @@ -3570,10 +3595,11 @@ fn gc_budgeted_step_work_units_inner_with_progress( // for. `js_gc_step_us` can only consult its clock BETWEEN units, so the // only honest statement about pause is a measured maximum. let step_started = std::time::Instant::now(); + let phase_code = cycle.state.phase().ffi_code(); let step = cycle.state.step(GcWorkBudget::bounded(work_units)); - super::instruments::note_budgeted_step_duration( - step_started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64, - ); + let step_us = step_started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64; + super::instruments::note_budgeted_step_duration(step_us); + super::diag_sites::budgeted_step_done(phase_code, step_us, work_units); super::instruments::note_incremental_step(); if step.completed { super::instruments::note_incremental_completion(); diff --git a/crates/perry-runtime/src/gc/survival_diag.rs b/crates/perry-runtime/src/gc/survival_diag.rs new file mode 100644 index 0000000000..af7f8fd64e --- /dev/null +++ b/crates/perry-runtime/src/gc/survival_diag.rs @@ -0,0 +1,240 @@ +//! `PERRY_GC_DIAG=1`: per copying minor, WHY each surviving byte survived. +//! +//! `[gc-copy-minor]` reports how much survived; it cannot say what kept it +//! alive. On the compiled claude-code TUI the streaming turn scavenges at +//! 57–99 % survival while a heap census puts the true live set at ~45 MB, so +//! at scavenge time something references almost the whole Eden that stops +//! referencing it soon after. The candidates differ in their fix — nepotism +//! through the remembered set (a dead-but-unswept old object whose dirty page +//! still points at young objects), a stack-map root, a registered side-table +//! scanner, a legitimately live render tree — and only attribution tells +//! them apart. +//! +//! Every object the copying minor moves or promotes is charged to the ORIGIN +//! that first reached it: +//! +//! * a direct root: the walk phase the collector is in +//! (`pin::copying_walk_phase()` — `mutable_root_slots/shadow_stack`, +//! `mutable_root_slots/native_stack`, `mutable_root_slots/global_root`, +//! or the registered scanner's name), and for the remembered set the OLD +//! PARENT'S type (`remembered_set/array`, `remembered_set/object`, …); +//! * a transitive reach: the origin of the worklist entry whose field scan +//! found it. The worklist carries a parallel origin vector so the drain +//! propagates it — the collector's own worklist is untouched. +//! +//! Output, after each minor's `[gc-copy-minor]` line: the top rows by bytes +//! as `[gc-survival] minor=N origin= type= objects= bytes= +//! promoted_bytes=`, then per-origin and per-type totals. Allocation is Rust +//! heap only (no JS-heap allocation inside the collector), and the whole +//! structure exists only while the diag is on — `CopyingNurseryCollector` +//! carries it as `Option>` and every hook is one null check. + +use super::*; +use std::collections::HashMap; + +#[derive(Default, Clone, Copy)] +struct Row { + objects: u64, + bytes: u64, + promoted_bytes: u64, +} + +pub(super) struct SurvivalDiag { + /// Interned origin names; the index is the origin id. + names: Vec, + /// `&'static str` identity → origin id, so a phase name is interned once. + by_ptr: HashMap, + /// Origin ids for `remembered_set/`, by parent `obj_type`. + remembered_ids: Vec, + /// The old parent's type while the remembered-set scan visits its slots. + pub(super) remembered_parent_type: u8, + /// Origin of the worklist entry currently being drained, if draining. + drain_origin: Option, + /// Parallel to `CopyingNurseryCollector::worklist`. + worklist_origin: Vec, + /// `(origin id << 8) | obj_type` → row. + rows: HashMap, +} + +impl SurvivalDiag { + pub(super) fn new() -> Self { + let mut d = Self { + names: Vec::new(), + by_ptr: HashMap::new(), + remembered_ids: Vec::new(), + remembered_parent_type: 0, + drain_origin: None, + worklist_origin: Vec::new(), + rows: HashMap::new(), + }; + for t in 0..=GC_TYPE_MAX as usize { + let name = gc_type_info(t as u8).map_or("?", |i| i.name); + let id = d.intern_owned(format!("remembered_set/{name}")); + d.remembered_ids.push(id); + } + d + } + + fn intern_owned(&mut self, name: String) -> u16 { + if let Some(i) = self.names.iter().position(|n| *n == name) { + return i as u16; + } + self.names.push(name); + (self.names.len() - 1) as u16 + } + + fn intern_static(&mut self, name: &'static str) -> u16 { + let key = name.as_ptr() as usize; + if let Some(&id) = self.by_ptr.get(&key) { + return id; + } + let id = self.intern_owned(name.to_string()); + self.by_ptr.insert(key, id); + id + } + + /// The origin a newly reached object is charged to right now. + fn current_origin(&mut self) -> u16 { + if let Some(o) = self.drain_origin { + return o; + } + let phase = super::pin::copying_walk_phase().unwrap_or("unknown"); + if phase == "remembered_set" { + let t = (self.remembered_parent_type as usize).min(self.remembered_ids.len() - 1); + return self.remembered_ids[t]; + } + self.intern_static(phase) + } + + /// Mirror of `collector.worklist.push(..)`. + #[inline] + pub(super) fn note_worklist_push(&mut self) { + let o = self.current_origin(); + self.worklist_origin.push(o); + } + + /// The drain is about to scan worklist entry `i`. + #[inline] + pub(super) fn begin_drain_entry(&mut self, i: usize) { + self.drain_origin = self.worklist_origin.get(i).copied(); + } + + pub(super) fn end_drain(&mut self) { + self.drain_origin = None; + } + + /// One object of `obj_type` and `bytes` was copied (or promoted). + #[inline] + pub(super) fn record(&mut self, obj_type: u8, bytes: usize, promoted: bool) { + let origin = self.current_origin(); + let key = (u32::from(origin) << 8) | u32::from(obj_type); + let row = self.rows.entry(key).or_default(); + row.objects += 1; + row.bytes += bytes as u64; + if promoted { + row.promoted_bytes += bytes as u64; + } + } + + pub(super) fn report(&self, seq: u64) { + #[cfg(test)] + LAST_REPORT.with(|r| { + *r.borrow_mut() = self + .rows + .iter() + .map(|(k, row)| { + ( + self.names[(k >> 8) as usize].clone(), + (k & 0xff) as u8, + row.objects, + row.bytes, + row.promoted_bytes, + ) + }) + .collect(); + }); + if self.rows.is_empty() { + return; + } + let mut rows: Vec<(u32, Row)> = self.rows.iter().map(|(k, r)| (*k, *r)).collect(); + rows.sort_by_key(|(_, r)| std::cmp::Reverse(r.bytes)); + let total_bytes: u64 = rows.iter().map(|(_, r)| r.bytes).sum(); + let total_objects: u64 = rows.iter().map(|(_, r)| r.objects).sum(); + eprintln!( + "[gc-survival] minor={seq} rows={} objects={total_objects} bytes={total_bytes}", + rows.len() + ); + for (key, r) in rows.iter().take(24) { + let origin = &self.names[(key >> 8) as usize]; + let t = (key & 0xff) as u8; + let tname = gc_type_info(t).map_or("?", |i| i.name); + eprintln!( + "[gc-survival] minor={seq} origin={origin} type={tname} objects={} bytes={} promoted_bytes={}", + r.objects, r.bytes, r.promoted_bytes + ); + } + let mut by_origin: HashMap = HashMap::new(); + let mut by_type: HashMap = HashMap::new(); + for (key, r) in &rows { + let o = by_origin.entry((key >> 8) as u16).or_default(); + o.objects += r.objects; + o.bytes += r.bytes; + o.promoted_bytes += r.promoted_bytes; + let t = by_type.entry((key & 0xff) as u8).or_default(); + t.objects += r.objects; + t.bytes += r.bytes; + t.promoted_bytes += r.promoted_bytes; + } + let mut by_origin: Vec<_> = by_origin.into_iter().collect(); + by_origin.sort_by_key(|(_, r)| std::cmp::Reverse(r.bytes)); + for (o, r) in by_origin.iter().take(12) { + eprintln!( + "[gc-survival] minor={seq} origin-total={} objects={} bytes={} permille={}", + self.names[*o as usize], + r.objects, + r.bytes, + if total_bytes > 0 { + r.bytes * 1000 / total_bytes + } else { + 0 + } + ); + } + let mut by_type: Vec<_> = by_type.into_iter().collect(); + by_type.sort_by_key(|(_, r)| std::cmp::Reverse(r.bytes)); + for (t, r) in by_type.iter().take(8) { + eprintln!( + "[gc-survival] minor={seq} type-total={} objects={} bytes={}", + gc_type_info(*t).map_or("?", |i| i.name), + r.objects, + r.bytes + ); + } + } +} + +crate::perry_thread_local! { + static MINOR_SEQ: Cell = const { Cell::new(0) }; +} + +#[cfg(test)] +crate::perry_thread_local! { + /// Test-only snapshot of the last report's rows: + /// `(origin, obj_type, objects, bytes, promoted_bytes)`. + static LAST_REPORT: RefCell> = const { RefCell::new(Vec::new()) }; +} + +/// Test-only: the rows of the most recent `report` on this thread. +#[cfg(test)] +pub(super) fn test_last_report() -> Vec<(String, u8, u64, u64, u64)> { + LAST_REPORT.with(|r| r.borrow().clone()) +} + +/// Sequence number for the next copying minor's report. +pub(super) fn next_minor_seq() -> u64 { + MINOR_SEQ.with(|c| { + let v = c.get() + 1; + c.set(v); + v + }) +} diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index 06d01002ac..ac62c3b605 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -15,10 +15,44 @@ pub const GC_RECENT_PAUSE_WINDOW: usize = 32; /// The value semantics are #5093's, shared with every other GC knob via /// [`super::env_flag_from_value`]. pub fn gc_diag_enabled() -> bool { + #[cfg(test)] + if GC_DIAG_TEST_FORCED.with(std::cell::Cell::get) { + return true; + } static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); *ENABLED.get_or_init(|| env_flag_enabled("PERRY_GC_DIAG")) } +#[cfg(test)] +thread_local! { + /// Test-only per-thread override of `PERRY_GC_DIAG`: the live reader is a + /// process-wide `OnceLock`, and `std::env::set_var` is shared by every + /// libtest thread (see `env_knob_parse.rs`), so a test that needs the + /// diagnostic paths live arms them here instead. + static GC_DIAG_TEST_FORCED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Test-only RAII: force `gc_diag_enabled()` ON for this thread. +#[cfg(test)] +pub(crate) struct GcDiagTestGuard { + previous: bool, +} + +#[cfg(test)] +impl GcDiagTestGuard { + pub(crate) fn force_on() -> Self { + let previous = GC_DIAG_TEST_FORCED.with(|c| c.replace(true)); + Self { previous } + } +} + +#[cfg(test)] +impl Drop for GcDiagTestGuard { + fn drop(&mut self) { + GC_DIAG_TEST_FORCED.with(|c| c.set(self.previous)); + } +} + /// Is `PERRY_GC_VERIFY_MARK` ON? Cached for the same reason as /// [`gc_diag_enabled`], and value-parsed for the same reason (#7991): the three /// mark-verifier call sites were presence-only, so `=0` armed a verifier that diff --git a/crates/perry-runtime/src/gc/tests/env_knob_parse.rs b/crates/perry-runtime/src/gc/tests/env_knob_parse.rs index 7f24d9ebcd..8a0bf964f4 100644 --- a/crates/perry-runtime/src/gc/tests/env_knob_parse.rs +++ b/crates/perry-runtime/src/gc/tests/env_knob_parse.rs @@ -50,6 +50,43 @@ const ON_SPELLINGS: &[&str] = &["1", "true", "on", "yes", "TRUE", "On", " 1 ", " /// default-OFF instrument OFF, not arm it. const UNRECOGNISED: &[&str] = &["banana", "2", "-1", "onn", "ye", "enabled", "0x1"]; +/// `PERRY_ALLOC_SITE_SAMPLE` (arena/alloc_sample.rs) is a MAGNITUDE knob with +/// the shared boolean vocabulary layered on top: every OFF spelling and every +/// typo reads as OFF, the ON spellings select the default interval, and an +/// explicit integer is the interval in bytes, floored. +#[test] +fn alloc_site_sample_interval_is_off_by_value_and_a_floored_magnitude_when_on() { + use crate::arena::alloc_sample::{parse_interval, DEFAULT_INTERVAL_BYTES, MIN_INTERVAL_BYTES}; + for raw in OFF_SPELLINGS { + assert_eq!(parse_interval(*raw), 0, "{raw:?} must read as OFF"); + } + for raw in ON_SPELLINGS { + assert_eq!( + parse_interval(Some(raw)), + DEFAULT_INTERVAL_BYTES, + "{raw:?} is the boolean ON spelling and selects the default interval" + ); + } + for raw in UNRECOGNISED { + if raw.trim().parse::().is_ok_and(|v| v >= 2) { + continue; // an integer is a magnitude for this knob, pinned below + } + assert_eq!( + parse_interval(Some(raw)), + 0, + "{raw:?} is a typo and must leave the sampler OFF" + ); + } + assert_eq!(parse_interval(Some("65536")), 65536); + assert_eq!(parse_interval(Some(" 4096 ")), 4096); + assert_eq!( + parse_interval(Some("2")), + MIN_INTERVAL_BYTES, + "a tiny explicit interval is floored, not honoured" + ); + assert!(DEFAULT_INTERVAL_BYTES >= MIN_INTERVAL_BYTES); +} + #[test] fn default_off_knobs_are_parsed_by_value_not_presence() { for raw in OFF_SPELLINGS { diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index ac6d08f01e..2d3300e1cd 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -57,6 +57,7 @@ mod shape_keys_descriptor_edge; mod smoke; mod step_bounds; pub(super) mod support; +mod survival_diag; mod teardown; mod telemetry_verifier; mod temp_roots; diff --git a/crates/perry-runtime/src/gc/tests/survival_diag.rs b/crates/perry-runtime/src/gc/tests/survival_diag.rs new file mode 100644 index 0000000000..7467984450 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/survival_diag.rs @@ -0,0 +1,143 @@ +//! `[gc-survival]` / `[gc-trigger]` / `[gc-full]` / `[gc-budgeted]` / +//! `[gc-charge]` (gc/survival_diag.rs, gc/diag_sites.rs): the attribution +//! instruments are validated against heaps and cycles of KNOWN shape before +//! they are pointed at anything real. Every assertion here can fail on the +//! instrument: a lost drain propagation charges elements to the drain phase, +//! a missed worklist mirror misaligns the origin vector, an unconsumed site +//! label mislabels the next full, an uncounted step leaves `steps` short. + +use super::super::*; +use super::support::*; + +/// A young array holding `N` young strings, rooted from ONE shadow-stack slot. +/// The elements are reachable only through the array, so their origin is the +/// array's — which is exactly the claim the parallel origin vector makes. +#[test] +fn survival_rows_charge_transitive_reach_to_the_originating_root() { + let _diag = crate::gc::telemetry::GcDiagTestGuard::force_on(); + let _guard = CopyingNurseryTestGuard::new(1); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + const N: usize = 40; + let mut arr = crate::array::js_array_alloc(N as u32); + for _ in 0..N { + let child = young_leaf(); + arr = crate::array::js_array_push_f64(arr, f64::from_bits(string_bits(child))); + } + js_shadow_slot_set(0, ptr_bits(arr as usize)); + + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false); + let moved = + trace.copying_nursery.copied_objects as u64 + trace.copying_nursery.promoted_objects as u64; + assert!( + moved > N as u64, + "subject must be live: the minor moved {moved} objects, expected at least {}", + N + 1 + ); + + let rows = super::super::survival_diag::test_last_report(); + assert!( + !rows.is_empty(), + "the diag was forced on, so the minor must have reported rows" + ); + let attributed: u64 = rows.iter().map(|r| r.2).sum(); + assert_eq!( + attributed, moved, + "every moved object is attributed exactly once (origin vector aligned with the worklist)" + ); + const SHADOW: &str = "mutable_root_slots/shadow_stack"; + let strings_via_shadow: u64 = rows + .iter() + .filter(|(o, t, ..)| o == SHADOW && *t == GC_TYPE_STRING) + .map(|r| r.2) + .sum(); + let arrays_via_shadow: u64 = rows + .iter() + .filter(|(o, t, ..)| o == SHADOW && *t == GC_TYPE_ARRAY) + .map(|r| r.2) + .sum(); + assert!( + arrays_via_shadow >= 1, + "the rooted array is charged to the shadow-stack root: rows={rows:?}" + ); + assert!( + strings_via_shadow >= N as u64, + "the {N} elements reach the collector only through the array, so they are charged to \ + the array's origin, not to the drain: rows={rows:?}" + ); + assert!( + rows.iter().all(|(o, ..)| !o.contains("worklist_drain")), + "transitive reach must never be charged to the drain phase: rows={rows:?}" + ); +} + +#[test] +fn full_site_label_is_consumed_once_and_counted_per_site() { + use super::super::diag_sites::*; + let _diag = crate::gc::telemetry::GcDiagTestGuard::force_on(); + set_full_site("survival_diag_test_a"); + assert_eq!(take_full_site(), "survival_diag_test_a"); + assert_eq!( + take_full_site(), + "sync", + "a label is consumed by the first full after it; the next full must not inherit it" + ); + let before = test_full_site_count("survival_diag_test_b"); + full_started("survival_diag_test_b", GcTriggerKind::Manual); + full_started("survival_diag_test_b", GcTriggerKind::OldGenBytes); + assert_eq!(test_full_site_count("survival_diag_test_b"), before + 2); + assert_eq!(test_full_site_count("survival_diag_test_never"), 0); +} + +#[test] +fn budgeted_accounting_counts_steps_and_root_scan_time() { + use super::super::diag_sites::*; + let _diag = crate::gc::telemetry::GcDiagTestGuard::force_on(); + budgeted_started( + GcTriggerKind::OldGenBytes, + GcCollectionKind::Full, + GcProgressKind::MutatorAssist, + ); + // Phase codes follow `GcCyclePhase::ffi_code`: 2 = root scan, 6 = sweep. + budgeted_step_done(2, 300, 16); + budgeted_step_done(2, 200, 16); + budgeted_step_done(6, 50, 16); + budgeted_completed(4096); + let (steps, step_us, units, root_us) = + test_last_budgeted().expect("a completed cycle publishes its accounting"); + assert_eq!(steps, 3); + assert_eq!(step_us, 550); + assert_eq!(units, 48); + assert_eq!( + root_us, 500, + "root-scan time is the sum of the steps taken in phase 2" + ); +} + +#[test] +fn charge_probe_attributes_only_under_the_diag() { + use super::super::diag_sites::*; + { + // Diag OFF: a probe is inert and records nothing. + let probe = ChargeProbe::begin(); + probe.end(7, ChargeKind::Assist); + } + let _diag = crate::gc::telemetry::GcDiagTestGuard::force_on(); + report_charges("survival_diag_test_reset"); + assert!( + test_charge_rows().is_empty(), + "report_charges drains the table" + ); + let probe = ChargeProbe::begin(); + probe.end(7, ChargeKind::SyncFull); + let probe = ChargeProbe::begin(); + probe.end(3, ChargeKind::Assist); + let rows = test_charge_rows(); + let calls: u64 = rows.iter().map(|r| r.0).sum(); + let units: u64 = rows.iter().map(|r| r.1).sum(); + let fulls: u64 = rows.iter().map(|r| r.3).sum(); + assert_eq!(calls, 2, "two probes ended under the diag: rows={rows:?}"); + assert_eq!(units, 10); + assert_eq!(fulls, 1); + report_charges("survival_diag_test_done"); +} diff --git a/crates/perry-runtime/src/gc/tests/young_log_tests.rs b/crates/perry-runtime/src/gc/tests/young_log_tests.rs index db66fadbf6..e6cf25887d 100644 --- a/crates/perry-runtime/src/gc/tests/young_log_tests.rs +++ b/crates/perry-runtime/src/gc/tests/young_log_tests.rs @@ -53,6 +53,11 @@ fn walk(table: &'static str) -> young_log::YoungLogWalk { young_log::last_walk(table).unwrap_or_else(|| panic!("no walk recorded for {table}")) } +/// For a table that is deliberately NOT young-logged: no walk row at all. +fn walk_opt(table: &'static str) -> Option { + young_log::last_walk(table) +} + // ---------------------------------------------------------------- closures #[test] @@ -430,15 +435,18 @@ fn young_transition_key_under_an_old_target_arms_the_log_through_the_writer() { ); } +/// The shape cache is deliberately NOT young-logged (see +/// `scan_shape_cache_roots_mut`): its keys arrays are longlived, so a log +/// there names every entry forever and skips nothing. This pins the walk that +/// replaced it — a young entry reachable only through the cache still moves +/// and is re-keyed in both the inline slot and the overflow map. #[test] -fn young_shape_cache_entry_is_moved_through_the_log() { +fn shape_cache_entry_is_moved_by_the_plain_walk() { let _guard = CopyingNurseryTestGuard::new(0); gc_register_mutable_root_scanner(crate::object::scan_shape_cache_roots_mut); - // Reachable ONLY through the cache (which roots it). Seeded through the - // PRODUCTION writer (`shape_cache_insert`), not a test seam: a seam that - // arms the log itself makes this test pass with the writer's own arm site - // deleted, which is how #9755 shipped an unenforced rule 1. + // Reachable ONLY through the cache (which roots it), seeded through the + // PRODUCTION writer (`shape_cache_insert`), not a test seam. let keys = unsafe { young_keys_array() }; let shape_id = 0x9754_0001; crate::object::test_shape_cache_insert(shape_id, keys); @@ -455,9 +463,11 @@ fn young_shape_cache_entry_is_moved_through_the_log() { inline, overflow, "inline and overflow must agree on the new address" ); - let row = walk("object.shape_cache"); - assert!(row.partial); - assert!(row.visited >= 1, "{row:?}"); + assert!( + walk_opt("object.shape_cache").is_none(), + "the shape cache must not report a young-log walk: #9755's log for it \ + skipped 0 % and cost 35 % more than this walk, and was removed" + ); } // --------------------------------------------------------------------------- diff --git a/crates/perry-runtime/src/object/alloc.rs b/crates/perry-runtime/src/object/alloc.rs index 0601ee7519..cd6d6e359a 100644 --- a/crates/perry-runtime/src/object/alloc.rs +++ b/crates/perry-runtime/src/object/alloc.rs @@ -915,6 +915,16 @@ pub unsafe extern "C" fn js_object_clone_with_extra( } let src_ptr = src_raw as *const ObjectHeader; + if super::string_wrapper::length(src_raw).is_some() { + let scope = crate::gc::RuntimeHandleScope::new(); + let src_h = scope.root_nanbox_f64(src_f64); + let target = js_object_alloc(0, 0); + let copied = js_object_assign_one( + crate::value::js_nanbox_pointer(target as i64), + src_h.get_nanbox_f64(), + ); + return crate::value::js_nanbox_get_pointer(copied) as *mut ObjectHeader; + } let src_field_count = crate::object::object_live_slot_count(src_ptr); // Physical slot capacity: src_field_count + extra_count, but at least max(fc, 8) to match @@ -1037,6 +1047,10 @@ pub unsafe extern "C" fn js_object_copy_own_fields(dst_i64: i64, src_f64: f64) { _ => return, } let src = src_raw as *const ObjectHeader; + if super::string_wrapper::length(src_raw).is_some() { + js_object_assign_one(crate::value::js_nanbox_pointer(dst as i64), src_f64); + return; + } // #6667: a native-module namespace (`{ ...require("crypto") }`) stores no // real fields — only the internal `__module__` sentinel — so the raw @@ -1811,7 +1825,14 @@ pub unsafe extern "C" fn js_object_assign_one(target_f64: f64, source_f64: f64) ); } } else if source_obj_type == crate::gc::GC_TYPE_OBJECT { - let src_keys = crate::object::object_keys_array(src); + let src_keys = if super::string_wrapper::length(src as usize).is_some() { + let names = src_h.with_const_ptr(|src: *const ObjectHeader| { + js_object_get_own_property_names(crate::value::js_nanbox_pointer(src as i64)) + }); + crate::value::js_nanbox_get_pointer(names) as *mut crate::ArrayHeader + } else { + crate::object::object_keys_array(src) + }; let keys_h = scope.root_raw_mut_ptr(src_keys); if !src_keys.is_null() && (src_keys as usize) >= 0x10000 { // Cap the key count at the keys array's capacity: a malformed keys diff --git a/crates/perry-runtime/src/object/delete_rest.rs b/crates/perry-runtime/src/object/delete_rest.rs index f8aeb554f9..cfa797bacd 100644 --- a/crates/perry-runtime/src/object/delete_rest.rs +++ b/crates/perry-runtime/src/object/delete_rest.rs @@ -227,6 +227,9 @@ pub extern "C" fn js_object_delete_field( } return 1; } + if super::string_wrapper::has_index_key(obj as usize, key) { + return 0; + } // Once #9064's stable marker is installed, this receiver has already // been proved to be an ordinary non-prototype object with no // descriptors. Avoid decoding the same dynamic key and scanning the @@ -1071,6 +1074,9 @@ pub extern "C" fn js_object_rest( return js_object_alloc(0, 0); } unsafe { + if super::string_wrapper::length(src as usize).is_some() { + return super::string_wrapper::rest(src, exclude_keys); + } let keys = crate::object::object_keys_array(src); if keys.is_null() { return js_object_alloc(0, 0); diff --git a/crates/perry-runtime/src/object/descriptor_state.rs b/crates/perry-runtime/src/object/descriptor_state.rs index 109ef1e75e..3e90299f56 100644 --- a/crates/perry-runtime/src/object/descriptor_state.rs +++ b/crates/perry-runtime/src/object/descriptor_state.rs @@ -130,7 +130,9 @@ impl DescriptorTables { const DESCRIPTOR_YOUNG_LOG_NAME: &str = "object.descriptors"; +mod gc_scan; mod young; +pub(crate) use gc_scan::{scan_descriptor_owner, scan_descriptor_roots_mut}; use young::{relevant_descriptor_owners, scan_descriptor_roots_young}; /// Rule 1 of `gc/young_log.rs`: log `owner` BEFORE its descriptor is @@ -558,18 +560,82 @@ pub(crate) fn note_descriptor_target(obj: usize) { /// Look up the property descriptor for (obj, key). Returns None if no entry exists, /// in which case the JS default `{ writable: true, enumerable: true, configurable: true }` applies. pub(crate) fn get_property_attrs(obj: usize, key: &str) -> Option { + // A STORED descriptor wins over the synthesized index default: + // `Object.defineProperty` / `Object.freeze` on a wrapper installs a real + // entry, and the §10.4.3 default must not shadow it. Synthesis therefore + // happens in the `string_wrapper_index_attrs` fallback BELOW the table + // probe, never as an early return above it. + // // #6759 Phase C2: the meta-record summary proves most misses without // the `String` build + table probe (and shields a fresh object at a // recycled address from a dead owner's not-yet-pruned entries). - if !may_have_descriptor_entry(obj, key, false) { + if may_have_descriptor_entry(obj, key, false) { + if let Some(attrs) = state() + .descriptors + .property_descriptors + .borrow() + .get(&(obj, key.to_string())) + .copied() + { + return Some(attrs); + } + } + string_wrapper_index_attrs(obj, key) +} + +/// ECMA-262 §10.4.3: every in-range integer index of a `String` exotic object +/// (`new String("abc")`, and the wrapper `ToObject` mints for a sloppy method +/// call on a string primitive) has the descriptor +/// `{ writable: false, enumerable: true, configurable: false }`. That is a +/// property of the CLASS and of the boxed length — never of the individual +/// object — so it is answered from the wrapper's own payload instead of being +/// stored once per character in `PROPERTY_DESCRIPTORS`. +/// +/// Storing it cost, per boxed character: a `String` key on the Rust heap, a +/// hash-map entry that only a full collection's dead-owner prune can reclaim, +/// an owner-index entry, a meta-descriptor key bit, and one program-wide +/// `prop_plan_epoch_bump()`. On the compiled claude-code TUI, whose render +/// path boxes a receiver per string method call, those entries were the +/// unbounded half of the process's resident growth during a turn. +/// +/// A REAL entry still wins (the probe above runs first): `Object.freeze` or +/// an explicit `defineProperty` on a wrapper installs one and is observed. +/// +/// The first byte is checked before anything else: an index key starts with an +/// ASCII digit, so every ordinary property name leaves through one compare. +#[inline] +fn string_wrapper_index_attrs(obj: usize, key: &str) -> Option { + let bytes = key.as_bytes(); + if !bytes.first().is_some_and(u8::is_ascii_digit) { return None; } - state() - .descriptors - .property_descriptors - .borrow() - .get(&(obj, key.to_string())) - .copied() + let index = canonical_index_key(bytes)?; + let len = crate::builtins::boxed_string_wrapper_utf16_len(obj)?; + (index < len).then(|| PropertyAttrs::new(false, true, false)) +} + +/// `CanonicalNumericIndexString` for the digits-only case: the key must be the +/// exact `ToString` of the integer it names, so `"0"` is an index but `"01"`, +/// `"1.0"` and `""` are not (mirrors `string::canonical_string_index`). +#[inline] +fn canonical_index_key(bytes: &[u8]) -> Option { + if bytes.is_empty() || bytes.len() > 10 { + return None; + } + if bytes[0] == b'0' { + return (bytes.len() == 1).then_some(0); + } + let mut value: u64 = 0; + for &b in bytes { + if !b.is_ascii_digit() { + return None; + } + value = value * 10 + (b - b'0') as u64; + if value > u32::MAX as u64 { + return None; + } + } + u32::try_from(value).ok() } /// Whether this specific object has ever had a property descriptor installed on @@ -1524,464 +1590,3 @@ fn rewrite_descriptor_owner( visitor.visit_metadata_usize_slot(&mut addr); addr } - -/// GC scanner for the string-keyed descriptor side tables (2026-07-02 audit -/// P0; ported from the stranded be73b4f8d): `ACCESSOR_DESCRIPTORS` holds the -/// ONLY reference to `Object.defineProperty` getter/setter closures (the -/// accessor install path stores no field-slot copy), so without visiting -/// them a minor GC sweeps or moves the closure out from under the next -/// property read. Owner keys are `(obj_addr, key)` — rekeyed when the owning -/// object moves, exactly like the symbol-keyed twins, so frozen/non-writable -/// attrs and accessors don't silently detach (or fire on a new tenant at a -/// reused address). -pub(crate) fn scan_descriptor_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - let st = state(); - // #9754: a minor-scoped pass visits only the young-logged owners; the - // full walk below rebuilds the log from what it finds. - if visitor.young_scope() { - scan_descriptor_roots_young(visitor, st); - return; - } - let table_len = st.descriptors.attr_keys_by_owner.borrow().len() as u64 - + st.descriptors.accessor_keys_by_owner.borrow().len() as u64; - { - // Probe DISTINCT OWNERS via the index, not every `(owner, key)` pair. - // This runs on every GC cycle, and since the moving young-gen scavenge - // became the default (#7019) that is often — so an O(total descriptors) - // probe here was a per-collection tax proportional to the whole - // program's descriptor count rather than to what actually moved. - let needs_rebuild = st - .descriptors - .attr_keys_by_owner - .borrow() - .keys() - .any(|owner| rewrite_descriptor_owner(visitor, *owner) != *owner); - let mut descriptors = st.descriptors.property_descriptors.borrow_mut(); - if needs_rebuild { - let old = std::mem::take(&mut *descriptors); - for ((owner, key), attrs) in old { - let owner = rewrite_descriptor_owner(visitor, owner); - descriptors.insert((owner, key), attrs); - } - } - } - - { - let needs_rebuild = st - .descriptors - .accessor_keys_by_owner - .borrow() - .keys() - .any(|owner| rewrite_descriptor_owner(visitor, *owner) != *owner); - let mut descriptors = st.descriptors.accessor_descriptors.borrow_mut(); - if needs_rebuild { - let old = std::mem::take(&mut *descriptors); - for ((owner, key), mut acc) in old { - if acc.get != 0 { - visitor.visit_nanbox_u64_slot(&mut acc.get); - } - if acc.set != 0 { - visitor.visit_nanbox_u64_slot(&mut acc.set); - } - let owner = rewrite_descriptor_owner(visitor, owner); - descriptors.insert((owner, key), acc); - } - } else { - for acc in descriptors.values_mut() { - if acc.get != 0 { - visitor.visit_nanbox_u64_slot(&mut acc.get); - } - if acc.set != 0 { - visitor.visit_nanbox_u64_slot(&mut acc.set); - } - } - } - } - - // Rekey the owner index itself. Evacuation moved the owning objects, so - // the tables above were rebuilt under new addresses; an index still keyed - // by the OLD addresses would report no keys for the moved object (silently - // dropping its accessors from `Object.keys`) and would keep a dead address - // alive in every later scan. Merge on collision: an address freed by one - // object can be reused by another in the same cycle. - for index in [ - &st.descriptors.attr_keys_by_owner, - &st.descriptors.accessor_keys_by_owner, - ] { - let mut idx = index.borrow_mut(); - if idx.is_empty() { - continue; - } - let needs_rekey = idx - .keys() - .any(|owner| rewrite_descriptor_owner(visitor, *owner) != *owner); - if !needs_rekey { - continue; - } - let old = std::mem::take(&mut *idx); - for (owner, keys) in old { - let owner = rewrite_descriptor_owner(visitor, owner); - let dest = idx.entry(owner).or_default(); - for k in keys { - if !dest.iter().any(|existing| *existing == k) { - dest.push(k); - } - } - } - } - - // A full walk is authoritative: rebuild the young log from the tables. - let kept = relevant_descriptor_owners(st); - let kept_len = kept.len() as u64; - { - let mut log = st.descriptors.young_owners.borrow_mut(); - let _ = log.take_sorted(); - log.extend(kept); - } - crate::gc::young_log::note_walk( - DESCRIPTOR_YOUNG_LOG_NAME, - crate::gc::young_log::YoungLogWalk { - partial: false, - logged: table_len, - visited: table_len, - kept: kept_len, - table_len, - }, - ); -} - -/// Visit one owner's descriptors. Returns the post-visit owner address and -/// whether the entry can still matter to a minor. -fn scan_descriptor_owner( - visitor: &mut crate::gc::RuntimeRootVisitor<'_>, - st: &crate::state::RuntimeState, - owner: usize, -) -> (usize, bool) { - use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; - let new_owner = rewrite_descriptor_owner(visitor, owner); - let mut relevant = false; - let accessor_keys = st - .descriptors - .accessor_keys_by_owner - .borrow() - .get(&owner) - .cloned() - .unwrap_or_default(); - if !accessor_keys.is_empty() { - let mut accessors = st.descriptors.accessor_descriptors.borrow_mut(); - for key in &accessor_keys { - if let Some(acc) = accessors.get_mut(&(owner, key.clone())) { - if acc.get != 0 { - visitor.visit_nanbox_u64_slot(&mut acc.get); - } - if acc.set != 0 { - visitor.visit_nanbox_u64_slot(&mut acc.set); - } - relevant |= bits_are_minor_relevant(acc.get) || bits_are_minor_relevant(acc.set); - } - } - if new_owner != owner { - for key in accessor_keys { - if let Some(acc) = accessors.remove(&(owner, key.clone())) { - accessors.insert((new_owner, key), acc); - } - } - } - } - if new_owner != owner { - let attr_keys = st - .descriptors - .attr_keys_by_owner - .borrow() - .get(&owner) - .cloned() - .unwrap_or_default(); - if !attr_keys.is_empty() { - let mut attrs = st.descriptors.property_descriptors.borrow_mut(); - for key in attr_keys { - if let Some(value) = attrs.remove(&(owner, key.clone())) { - attrs.insert((new_owner, key), value); - } - } - } - owner_index_transfer(&st.descriptors.attr_keys_by_owner, owner, new_owner); - owner_index_transfer(&st.descriptors.accessor_keys_by_owner, owner, new_owner); - } - relevant |= addr_is_minor_relevant(new_owner); - (new_owner, relevant) -} - -/// The owner index (`attr_keys_by_owner` / `accessor_keys_by_owner`) exists -/// only to answer "which keys does this owner have?" without walking every -/// descriptor in the process. It is a mirror, so the one way it can break is -/// **drift** from the tables it mirrors — which would not crash, it would -/// silently drop keys from `Object.keys` or resurrect deleted ones. -/// -/// These tests therefore assert the mirror invariant directly (index == -/// what a full scan of the table would return) across install, redefine, -/// delete, bulk-clear and owner-transfer. -#[cfg(test)] -mod owner_index_tests { - use super::*; - use std::collections::BTreeSet; - - /// What the pre-index implementation would have computed: a full scan of - /// the table filtered by owner. The index must always agree with this. - fn scan_table_keys(accessor: bool, owner: usize) -> BTreeSet { - let st = state(); - if accessor { - st.descriptors - .accessor_descriptors - .borrow() - .keys() - .filter(|(o, _)| *o == owner) - .map(|(_, k)| k.clone()) - .collect() - } else { - st.descriptors - .property_descriptors - .borrow() - .keys() - .filter(|(o, _)| *o == owner) - .map(|(_, k)| k.clone()) - .collect() - } - } - - fn index_keys(accessor: bool, owner: usize) -> BTreeSet { - let st = state(); - let idx = if accessor { - &st.descriptors.accessor_keys_by_owner - } else { - &st.descriptors.attr_keys_by_owner - }; - idx.borrow() - .get(&owner) - .cloned() - .unwrap_or_default() - .into_iter() - .collect() - } - - fn assert_mirrors(owner: usize, ctx: &str) { - for (accessor, label) in [(false, "property"), (true, "accessor")] { - assert_eq!( - index_keys(accessor, owner), - scan_table_keys(accessor, owner), - "{label} owner index drifted from the table it mirrors ({ctx}); \ - a drift here silently corrupts Object.keys / for-in output" - ); - } - } - - #[test] - fn index_mirrors_tables_across_install_redefine_and_delete() { - let _lock = crate::gc::global_side_table_test_lock(); - let obj = crate::object::js_object_alloc(0, 0); - let addr = obj as usize; - - set_property_attrs(addr, "a".to_string(), PropertyAttrs::new(true, true, true)); - set_property_attrs(addr, "b".to_string(), PropertyAttrs::new(true, true, true)); - set_accessor_descriptor(addr, "g".to_string(), AccessorDescriptor::default()); - assert_mirrors(addr, "after installs"); - - // Redefining an existing key must not duplicate it — a duplicate would - // make `Object.keys` report the key twice. - set_property_attrs(addr, "a".to_string(), PropertyAttrs::new(true, true, true)); - set_accessor_descriptor(addr, "g".to_string(), AccessorDescriptor::default()); - assert_eq!( - state() - .descriptors - .attr_keys_by_owner - .borrow() - .get(&addr) - .map(|v| v.len()), - Some(2), - "redefining an existing descriptor must not push a duplicate key" - ); - assert_mirrors(addr, "after redefine"); - - clear_property_attrs(addr, "a"); - clear_accessor_descriptor(addr, "g"); - assert_mirrors(addr, "after delete"); - - // Deleting the last key must drop the owner entry entirely, so a dead - // owner leaves nothing for later GC scans to walk. - clear_property_attrs(addr, "b"); - assert!( - !state() - .descriptors - .attr_keys_by_owner - .borrow() - .contains_key(&addr), - "an owner with no remaining descriptors must be removed from the index" - ); - } - - #[test] - fn accessor_keys_for_obj_agrees_with_a_full_scan() { - let _lock = crate::gc::global_side_table_test_lock(); - let obj = crate::object::js_object_alloc(0, 0); - let addr = obj as usize; - // A second owner with its own accessors: the whole point of the index - // is that this one's keys never leak into the first one's answer. - let other = crate::object::js_object_alloc(0, 0); - let other_addr = other as usize; - - for k in ["z", "m", "a"] { - set_accessor_descriptor(addr, k.to_string(), AccessorDescriptor::default()); - } - for k in ["zz", "mm"] { - set_accessor_descriptor(other_addr, k.to_string(), AccessorDescriptor::default()); - } - - let got = accessor_descriptor_keys_for_obj(addr); - assert_eq!( - got, - vec!["a".to_string(), "m".to_string(), "z".to_string()], - "keys must be sorted and scoped to the requested owner only" - ); - assert_eq!( - got.into_iter().collect::>(), - scan_table_keys(true, addr), - "the index answer must equal what a full table scan would return" - ); - } - - #[test] - fn transfer_moves_both_tables_and_the_index() { - let _lock = crate::gc::global_side_table_test_lock(); - let old = crate::object::js_object_alloc(0, 0) as usize; - let new = crate::object::js_object_alloc(0, 0) as usize; - - set_property_attrs(old, "p".to_string(), PropertyAttrs::new(true, true, true)); - set_accessor_descriptor(old, "acc".to_string(), AccessorDescriptor::default()); - - transfer_descriptor_owner(old, new); - - assert_mirrors(old, "old owner after transfer"); - assert_mirrors(new, "new owner after transfer"); - assert!( - scan_table_keys(false, old).is_empty() && scan_table_keys(true, old).is_empty(), - "transfer must leave nothing behind under the old owner address" - ); - assert_eq!( - accessor_descriptor_keys_for_obj(new), - vec!["acc".to_string()], - "accessors must be readable through the new owner address after growth" - ); - } - - #[test] - fn clear_object_descriptors_empties_the_index_too() { - let _lock = crate::gc::global_side_table_test_lock(); - let obj = crate::object::js_object_alloc(0, 0) as usize; - // `clear_object_descriptors` early-returns unless a handle-band owner - // has ever taken a descriptor; set the latch so the body actually runs. - HANDLE_HAS_DESCRIPTORS.store(true, Ordering::Relaxed); - - set_property_attrs(obj, "p".to_string(), PropertyAttrs::new(true, true, true)); - set_accessor_descriptor(obj, "acc".to_string(), AccessorDescriptor::default()); - assert_mirrors(obj, "before clear"); - - clear_object_descriptors(obj); - assert_mirrors(obj, "after clear"); - assert!( - accessor_descriptor_keys_for_obj(obj).is_empty(), - "a cleared owner must report no accessor keys" - ); - } -} - -#[cfg(test)] -mod c5a_tests { - use super::*; - - /// #6759 C5a: a prototype-level descriptor whose key names no declared - /// instance field must NOT flip the process-wide inline-guard disable; - /// one whose key IS a declared field must. - #[test] - fn inline_guard_disable_is_per_declared_field_key() { - let _lock = crate::gc::global_side_table_test_lock(); - test_reset_class_field_inline_guard(); - - let proto = crate::object::js_object_alloc(0, 0); - class_registry::class_prototype_object_root_store(0x0666_0001, proto); - let proto_addr = proto as usize; - - // Method-style install (babel output): key declared by no class. - set_accessor_descriptor( - proto_addr, - "c5a_render_method".to_string(), - AccessorDescriptor::default(), - ); - assert!( - class_field_inline_guard_enabled(), - "a prototype install keyed by a non-field name must not poison \ - the inline class-field fast path" - ); - assert!( - !class_registry::class_prototype_fast_guards_invalidated(), - "a keyed prototype descriptor must not retire every method guard" - ); - let render_slot = class_registry::class_prototype_method_guard_slot("c5a_render_method"); - assert!( - class_registry::class_prototype_fast_guard_invalidated_for_method(render_slot), - "a prototype descriptor must retire its matching method guard" - ); - let other_slot = class_registry::class_prototype_method_guard_slot("c5a_other_method"); - assert!( - !class_registry::class_prototype_fast_guard_invalidated_for_method(other_slot), - "an unrelated method guard must remain valid" - ); - - // Field-style install: key declared by a registered class. - note_declared_instance_field_name(b"c5a_field_x"); - assert!( - class_field_inline_guard_enabled(), - "declaring the field alone (no matching install) must not disable" - ); - set_property_attrs( - proto_addr, - "c5a_field_x".to_string(), - PropertyAttrs::new(false, true, true), - ); - assert!( - !class_field_inline_guard_enabled(), - "a prototype install keyed by a DECLARED field must disable" - ); - - test_reset_class_field_inline_guard(); - } - - /// #6759 C5a ordering: an install that precedes the declaring class's - /// registration is retro-checked when the class arrives. - #[test] - fn inline_guard_retro_disable_on_late_class_registration() { - let _lock = crate::gc::global_side_table_test_lock(); - test_reset_class_field_inline_guard(); - - let proto = crate::object::js_object_alloc(0, 0); - class_registry::class_prototype_object_root_store(0x0666_0002, proto); - - set_accessor_descriptor( - proto as usize, - "c5a_late_field".to_string(), - AccessorDescriptor::default(), - ); - assert!( - class_field_inline_guard_enabled(), - "no class declares the key yet — install must skip the disable" - ); - - // The declaring class registers AFTER the install. - note_declared_instance_field_name(b"c5a_late_field"); - assert!( - !class_field_inline_guard_enabled(), - "late class registration must retro-trigger the disable for \ - prototype keys installed earlier" - ); - - test_reset_class_field_inline_guard(); - } -} diff --git a/crates/perry-runtime/src/object/descriptor_state/gc_scan.rs b/crates/perry-runtime/src/object/descriptor_state/gc_scan.rs new file mode 100644 index 0000000000..fe0f22ca36 --- /dev/null +++ b/crates/perry-runtime/src/object/descriptor_state/gc_scan.rs @@ -0,0 +1,555 @@ +//! Descriptor-table GC root scanning. +//! +//! Split out of `descriptor_state.rs` to keep that file under the 2000-line +//! size gate. `scan_descriptor_roots_mut` is registered in `gc/mod.rs`. + +use super::*; + +/// GC scanner for the string-keyed descriptor side tables (2026-07-02 audit +/// P0; ported from the stranded be73b4f8d): `ACCESSOR_DESCRIPTORS` holds the +/// ONLY reference to `Object.defineProperty` getter/setter closures (the +/// accessor install path stores no field-slot copy), so without visiting +/// them a minor GC sweeps or moves the closure out from under the next +/// property read. Owner keys are `(obj_addr, key)` — rekeyed when the owning +/// object moves, exactly like the symbol-keyed twins, so frozen/non-writable +/// attrs and accessors don't silently detach (or fire on a new tenant at a +/// reused address). +pub(crate) fn scan_descriptor_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let st = state(); + // #9754: a minor-scoped pass visits only the young-logged owners; the + // full walk below rebuilds the log from what it finds. + if visitor.young_scope() { + scan_descriptor_roots_young(visitor, st); + return; + } + let table_len = st.descriptors.attr_keys_by_owner.borrow().len() as u64 + + st.descriptors.accessor_keys_by_owner.borrow().len() as u64; + { + // Probe DISTINCT OWNERS via the index, not every `(owner, key)` pair. + // This runs on every GC cycle, and since the moving young-gen scavenge + // became the default (#7019) that is often — so an O(total descriptors) + // probe here was a per-collection tax proportional to the whole + // program's descriptor count rather than to what actually moved. + let needs_rebuild = st + .descriptors + .attr_keys_by_owner + .borrow() + .keys() + .any(|owner| rewrite_descriptor_owner(visitor, *owner) != *owner); + let mut descriptors = st.descriptors.property_descriptors.borrow_mut(); + if needs_rebuild { + let old = std::mem::take(&mut *descriptors); + for ((owner, key), attrs) in old { + let owner = rewrite_descriptor_owner(visitor, owner); + descriptors.insert((owner, key), attrs); + } + } + } + + { + let needs_rebuild = st + .descriptors + .accessor_keys_by_owner + .borrow() + .keys() + .any(|owner| rewrite_descriptor_owner(visitor, *owner) != *owner); + let mut descriptors = st.descriptors.accessor_descriptors.borrow_mut(); + if needs_rebuild { + let old = std::mem::take(&mut *descriptors); + for ((owner, key), mut acc) in old { + if acc.get != 0 { + visitor.visit_nanbox_u64_slot(&mut acc.get); + } + if acc.set != 0 { + visitor.visit_nanbox_u64_slot(&mut acc.set); + } + let owner = rewrite_descriptor_owner(visitor, owner); + descriptors.insert((owner, key), acc); + } + } else { + for acc in descriptors.values_mut() { + if acc.get != 0 { + visitor.visit_nanbox_u64_slot(&mut acc.get); + } + if acc.set != 0 { + visitor.visit_nanbox_u64_slot(&mut acc.set); + } + } + } + } + + // Rekey the owner index itself. Evacuation moved the owning objects, so + // the tables above were rebuilt under new addresses; an index still keyed + // by the OLD addresses would report no keys for the moved object (silently + // dropping its accessors from `Object.keys`) and would keep a dead address + // alive in every later scan. Merge on collision: an address freed by one + // object can be reused by another in the same cycle. + for index in [ + &st.descriptors.attr_keys_by_owner, + &st.descriptors.accessor_keys_by_owner, + ] { + let mut idx = index.borrow_mut(); + if idx.is_empty() { + continue; + } + let needs_rekey = idx + .keys() + .any(|owner| rewrite_descriptor_owner(visitor, *owner) != *owner); + if !needs_rekey { + continue; + } + let old = std::mem::take(&mut *idx); + for (owner, keys) in old { + let owner = rewrite_descriptor_owner(visitor, owner); + let dest = idx.entry(owner).or_default(); + for k in keys { + if !dest.iter().any(|existing| *existing == k) { + dest.push(k); + } + } + } + } + + // A full walk is authoritative: rebuild the young log from the tables. + let kept = relevant_descriptor_owners(st); + let kept_len = kept.len() as u64; + { + let mut log = st.descriptors.young_owners.borrow_mut(); + let _ = log.take_sorted(); + log.extend(kept); + } + crate::gc::young_log::note_walk( + DESCRIPTOR_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: table_len, + visited: table_len, + kept: kept_len, + table_len, + }, + ); +} + +/// Visit one owner's descriptors. Returns the post-visit owner address and +/// whether the entry can still matter to a minor. +pub(crate) fn scan_descriptor_owner( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, + st: &crate::state::RuntimeState, + owner: usize, +) -> (usize, bool) { + use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + let new_owner = rewrite_descriptor_owner(visitor, owner); + let mut relevant = false; + let accessor_keys = st + .descriptors + .accessor_keys_by_owner + .borrow() + .get(&owner) + .cloned() + .unwrap_or_default(); + if !accessor_keys.is_empty() { + let mut accessors = st.descriptors.accessor_descriptors.borrow_mut(); + for key in &accessor_keys { + if let Some(acc) = accessors.get_mut(&(owner, key.clone())) { + if acc.get != 0 { + visitor.visit_nanbox_u64_slot(&mut acc.get); + } + if acc.set != 0 { + visitor.visit_nanbox_u64_slot(&mut acc.set); + } + relevant |= bits_are_minor_relevant(acc.get) || bits_are_minor_relevant(acc.set); + } + } + if new_owner != owner { + for key in accessor_keys { + if let Some(acc) = accessors.remove(&(owner, key.clone())) { + accessors.insert((new_owner, key), acc); + } + } + } + } + if new_owner != owner { + let attr_keys = st + .descriptors + .attr_keys_by_owner + .borrow() + .get(&owner) + .cloned() + .unwrap_or_default(); + if !attr_keys.is_empty() { + let mut attrs = st.descriptors.property_descriptors.borrow_mut(); + for key in attr_keys { + if let Some(value) = attrs.remove(&(owner, key.clone())) { + attrs.insert((new_owner, key), value); + } + } + } + owner_index_transfer(&st.descriptors.attr_keys_by_owner, owner, new_owner); + owner_index_transfer(&st.descriptors.accessor_keys_by_owner, owner, new_owner); + } + relevant |= addr_is_minor_relevant(new_owner); + (new_owner, relevant) +} + +/// The owner index (`attr_keys_by_owner` / `accessor_keys_by_owner`) exists +/// only to answer "which keys does this owner have?" without walking every +/// descriptor in the process. It is a mirror, so the one way it can break is +/// **drift** from the tables it mirrors — which would not crash, it would +/// silently drop keys from `Object.keys` or resurrect deleted ones. +/// +/// These tests therefore assert the mirror invariant directly (index == +/// what a full scan of the table would return) across install, redefine, +/// delete, bulk-clear and owner-transfer. +#[cfg(test)] +mod owner_index_tests { + use super::*; + use std::collections::BTreeSet; + + /// What the pre-index implementation would have computed: a full scan of + /// the table filtered by owner. The index must always agree with this. + fn scan_table_keys(accessor: bool, owner: usize) -> BTreeSet { + let st = state(); + if accessor { + st.descriptors + .accessor_descriptors + .borrow() + .keys() + .filter(|(o, _)| *o == owner) + .map(|(_, k)| k.clone()) + .collect() + } else { + st.descriptors + .property_descriptors + .borrow() + .keys() + .filter(|(o, _)| *o == owner) + .map(|(_, k)| k.clone()) + .collect() + } + } + + fn index_keys(accessor: bool, owner: usize) -> BTreeSet { + let st = state(); + let idx = if accessor { + &st.descriptors.accessor_keys_by_owner + } else { + &st.descriptors.attr_keys_by_owner + }; + idx.borrow() + .get(&owner) + .cloned() + .unwrap_or_default() + .into_iter() + .collect() + } + + fn assert_mirrors(owner: usize, ctx: &str) { + for (accessor, label) in [(false, "property"), (true, "accessor")] { + assert_eq!( + index_keys(accessor, owner), + scan_table_keys(accessor, owner), + "{label} owner index drifted from the table it mirrors ({ctx}); \ + a drift here silently corrupts Object.keys / for-in output" + ); + } + } + + #[test] + fn index_mirrors_tables_across_install_redefine_and_delete() { + let _lock = crate::gc::global_side_table_test_lock(); + let obj = crate::object::js_object_alloc(0, 0); + let addr = obj as usize; + + set_property_attrs(addr, "a".to_string(), PropertyAttrs::new(true, true, true)); + set_property_attrs(addr, "b".to_string(), PropertyAttrs::new(true, true, true)); + set_accessor_descriptor(addr, "g".to_string(), AccessorDescriptor::default()); + assert_mirrors(addr, "after installs"); + + // Redefining an existing key must not duplicate it — a duplicate would + // make `Object.keys` report the key twice. + set_property_attrs(addr, "a".to_string(), PropertyAttrs::new(true, true, true)); + set_accessor_descriptor(addr, "g".to_string(), AccessorDescriptor::default()); + assert_eq!( + state() + .descriptors + .attr_keys_by_owner + .borrow() + .get(&addr) + .map(|v| v.len()), + Some(2), + "redefining an existing descriptor must not push a duplicate key" + ); + assert_mirrors(addr, "after redefine"); + + clear_property_attrs(addr, "a"); + clear_accessor_descriptor(addr, "g"); + assert_mirrors(addr, "after delete"); + + // Deleting the last key must drop the owner entry entirely, so a dead + // owner leaves nothing for later GC scans to walk. + clear_property_attrs(addr, "b"); + assert!( + !state() + .descriptors + .attr_keys_by_owner + .borrow() + .contains_key(&addr), + "an owner with no remaining descriptors must be removed from the index" + ); + } + + #[test] + fn accessor_keys_for_obj_agrees_with_a_full_scan() { + let _lock = crate::gc::global_side_table_test_lock(); + let obj = crate::object::js_object_alloc(0, 0); + let addr = obj as usize; + // A second owner with its own accessors: the whole point of the index + // is that this one's keys never leak into the first one's answer. + let other = crate::object::js_object_alloc(0, 0); + let other_addr = other as usize; + + for k in ["z", "m", "a"] { + set_accessor_descriptor(addr, k.to_string(), AccessorDescriptor::default()); + } + for k in ["zz", "mm"] { + set_accessor_descriptor(other_addr, k.to_string(), AccessorDescriptor::default()); + } + + let got = accessor_descriptor_keys_for_obj(addr); + assert_eq!( + got, + vec!["a".to_string(), "m".to_string(), "z".to_string()], + "keys must be sorted and scoped to the requested owner only" + ); + assert_eq!( + got.into_iter().collect::>(), + scan_table_keys(true, addr), + "the index answer must equal what a full table scan would return" + ); + } + + #[test] + fn transfer_moves_both_tables_and_the_index() { + let _lock = crate::gc::global_side_table_test_lock(); + let old = crate::object::js_object_alloc(0, 0) as usize; + let new = crate::object::js_object_alloc(0, 0) as usize; + + set_property_attrs(old, "p".to_string(), PropertyAttrs::new(true, true, true)); + set_accessor_descriptor(old, "acc".to_string(), AccessorDescriptor::default()); + + transfer_descriptor_owner(old, new); + + assert_mirrors(old, "old owner after transfer"); + assert_mirrors(new, "new owner after transfer"); + assert!( + scan_table_keys(false, old).is_empty() && scan_table_keys(true, old).is_empty(), + "transfer must leave nothing behind under the old owner address" + ); + assert_eq!( + accessor_descriptor_keys_for_obj(new), + vec!["acc".to_string()], + "accessors must be readable through the new owner address after growth" + ); + } + + #[test] + fn clear_object_descriptors_empties_the_index_too() { + let _lock = crate::gc::global_side_table_test_lock(); + let obj = crate::object::js_object_alloc(0, 0) as usize; + // `clear_object_descriptors` early-returns unless a handle-band owner + // has ever taken a descriptor; set the latch so the body actually runs. + HANDLE_HAS_DESCRIPTORS.store(true, Ordering::Relaxed); + + set_property_attrs(obj, "p".to_string(), PropertyAttrs::new(true, true, true)); + set_accessor_descriptor(obj, "acc".to_string(), AccessorDescriptor::default()); + assert_mirrors(obj, "before clear"); + + clear_object_descriptors(obj); + assert_mirrors(obj, "after clear"); + assert!( + accessor_descriptor_keys_for_obj(obj).is_empty(), + "a cleared owner must report no accessor keys" + ); + } +} + +#[cfg(test)] +mod c5a_tests { + use super::*; + + /// #6759 C5a: a prototype-level descriptor whose key names no declared + /// instance field must NOT flip the process-wide inline-guard disable; + /// one whose key IS a declared field must. + #[test] + fn inline_guard_disable_is_per_declared_field_key() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset_class_field_inline_guard(); + + let proto = crate::object::js_object_alloc(0, 0); + class_registry::class_prototype_object_root_store(0x0666_0001, proto); + let proto_addr = proto as usize; + + // Method-style install (babel output): key declared by no class. + set_accessor_descriptor( + proto_addr, + "c5a_render_method".to_string(), + AccessorDescriptor::default(), + ); + assert!( + class_field_inline_guard_enabled(), + "a prototype install keyed by a non-field name must not poison \ + the inline class-field fast path" + ); + assert!( + !class_registry::class_prototype_fast_guards_invalidated(), + "a keyed prototype descriptor must not retire every method guard" + ); + let render_slot = class_registry::class_prototype_method_guard_slot("c5a_render_method"); + assert!( + class_registry::class_prototype_fast_guard_invalidated_for_method(render_slot), + "a prototype descriptor must retire its matching method guard" + ); + let other_slot = class_registry::class_prototype_method_guard_slot("c5a_other_method"); + assert!( + !class_registry::class_prototype_fast_guard_invalidated_for_method(other_slot), + "an unrelated method guard must remain valid" + ); + + // Field-style install: key declared by a registered class. + note_declared_instance_field_name(b"c5a_field_x"); + assert!( + class_field_inline_guard_enabled(), + "declaring the field alone (no matching install) must not disable" + ); + set_property_attrs( + proto_addr, + "c5a_field_x".to_string(), + PropertyAttrs::new(false, true, true), + ); + assert!( + !class_field_inline_guard_enabled(), + "a prototype install keyed by a DECLARED field must disable" + ); + + test_reset_class_field_inline_guard(); + } + + /// #6759 C5a ordering: an install that precedes the declaring class's + /// registration is retro-checked when the class arrives. + #[test] + fn inline_guard_retro_disable_on_late_class_registration() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset_class_field_inline_guard(); + + let proto = crate::object::js_object_alloc(0, 0); + class_registry::class_prototype_object_root_store(0x0666_0002, proto); + + set_accessor_descriptor( + proto as usize, + "c5a_late_field".to_string(), + AccessorDescriptor::default(), + ); + assert!( + class_field_inline_guard_enabled(), + "no class declares the key yet — install must skip the disable" + ); + + // The declaring class registers AFTER the install. + note_declared_instance_field_name(b"c5a_late_field"); + assert!( + !class_field_inline_guard_enabled(), + "late class registration must retro-trigger the disable for \ + prototype keys installed earlier" + ); + + test_reset_class_field_inline_guard(); + } +} + +#[cfg(test)] +pub(crate) fn test_property_descriptor_entry_count(obj: usize) -> usize { + state() + .descriptors + .property_descriptors + .borrow() + .keys() + .filter(|(owner, _)| *owner == obj) + .count() +} + +#[cfg(test)] +mod string_wrapper_index_attrs_tests { + use super::*; + + fn boxed(text: &str) -> usize { + let s = crate::string::js_string_from_bytes(text.as_ptr(), text.len() as u32); + let value = f64::from_bits(crate::value::JSValue::string_ptr(s).bits()); + let boxed = crate::builtins::js_boxed_string_new(value, 1); + crate::value::js_nanbox_get_pointer(boxed) as usize + } + + /// The index descriptors of a `String` exotic object are answered from the + /// wrapper's payload, not from `PROPERTY_DESCRIPTORS`. Both halves matter: + /// the ANSWER must still be the spec's + /// `{ writable: false, enumerable: true, configurable: false }` (delete + /// this synthesis and `str[0] = "x"` starts mutating the wrapper), and the + /// STORAGE must be one entry — `length` — however long the string is + /// (that is the allocation this exists to remove). + #[test] + fn in_range_indices_are_synthesized_and_not_stored() { + let obj = boxed("hello world"); + for index in ["0", "1", "10"] { + let attrs = get_property_attrs(obj, index) + .unwrap_or_else(|| panic!("index {index} must have a descriptor")); + assert!(!attrs.writable(), "index {index} is not writable"); + assert!(attrs.enumerable(), "index {index} is enumerable"); + assert!(!attrs.configurable(), "index {index} is not configurable"); + } + assert_eq!( + test_property_descriptor_entry_count(obj), + 1, + "only `length` is stored; the 11 index descriptors are synthesized" + ); + } + + /// Out of range, non-canonical, and non-index keys get the ordinary + /// answer, so the synthesis cannot invent properties the object does not + /// have. `"01"` and `"1.0"` are NOT canonical index strings. + #[test] + fn only_canonical_in_range_indices_are_synthesized() { + let obj = boxed("abc"); + assert!(get_property_attrs(obj, "3").is_none(), "past the end"); + assert!(get_property_attrs(obj, "01").is_none(), "not canonical"); + assert!(get_property_attrs(obj, "1.0").is_none(), "not canonical"); + assert!(get_property_attrs(obj, "").is_none()); + assert!(get_property_attrs(obj, "toString").is_none()); + assert!( + get_property_attrs(obj, "0").is_some(), + "the positive control: the same call answers for a real index" + ); + } + + /// Nothing but a String wrapper answers. A plain object with an index-named + /// property keeps the JS default (writable, enumerable, configurable), which + /// is what `None` means to every caller. + #[test] + fn a_plain_object_is_never_treated_as_a_string_wrapper() { + let obj = crate::object::js_object_alloc(0, 1) as usize; + let key = crate::string::js_string_from_bytes(b"0".as_ptr(), 1); + crate::object::js_object_set_field_by_name(obj as *mut _, key, 1.0); + assert!(get_property_attrs(obj, "0").is_none()); + assert!(get_property_attrs(0, "0").is_none(), "null address"); + } + + /// A REAL entry still wins: `Object.defineProperty` / `Object.freeze` on a + /// wrapper installs one, and the synthesized default must not shadow it. + #[test] + fn a_stored_descriptor_overrides_the_synthesized_one() { + let obj = boxed("xy"); + set_property_attrs(obj, "1".to_string(), PropertyAttrs::new(true, false, true)); + let attrs = get_property_attrs(obj, "1").expect("stored entry"); + assert!(attrs.writable() && !attrs.enumerable() && attrs.configurable()); + let other = get_property_attrs(obj, "0").expect("synthesized entry"); + assert!(!other.writable() && other.enumerable() && !other.configurable()); + } +} diff --git a/crates/perry-runtime/src/object/descriptors.rs b/crates/perry-runtime/src/object/descriptors.rs index f3bb2f62cc..a17edbc176 100644 --- a/crates/perry-runtime/src/object/descriptors.rs +++ b/crates/perry-runtime/src/object/descriptors.rs @@ -89,12 +89,18 @@ unsafe fn boxed_string_own_property_names(obj_value: f64, str_value: f64) -> f64 } sort_property_names_ecma(&mut names); + let scope = crate::gc::RuntimeHandleScope::new(); let result = crate::array::js_array_alloc(names.len() as u32); + let result_h = scope.root_raw_mut_ptr(result); for name in names { let str_ptr = crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32); - crate::array::js_array_push(result, JSValue::string_ptr(str_ptr)); + result_h.with_mut_ptr(|result| { + crate::array::js_array_push(result, JSValue::string_ptr(str_ptr)) + }); } - f64::from_bits((result as u64) | 0x7FFD_0000_0000_0000) + result_h.with_mut_ptr(|result: *mut crate::ArrayHeader| { + f64::from_bits(JSValue::array_ptr(result).bits()) + }) } /// Object.getOwnPropertyDescriptor(obj, key) — returns a data descriptor @@ -1214,16 +1220,15 @@ unsafe fn string_primitive_descriptor(str_value: f64, key_value: f64) -> f64 { if let Some(index) = super::canonical_array_index(name) { if index < utf16_len { - // Materialize the single UTF-16 unit at `index` as a 1-char string. - let bytes = std::slice::from_raw_parts(sptr, sblen as usize); - let s = std::str::from_utf8(bytes).unwrap_or(""); - if let Some(ch) = s.chars().nth(index as usize) { - let mut buf = [0u8; 4]; - let cs = ch.encode_utf8(&mut buf); - let cstr = crate::string::js_string_from_bytes(cs.as_ptr(), cs.len() as u32); - let char_val = f64::from_bits(JSValue::string_ptr(cstr).bits()); - return build_data_descriptor(char_val, false, true, false); - } + // String exotic indices are UTF-16 code units, including lone + // surrogate halves. Use the same read path as s[index]; `.chars()` + // counts Unicode scalars and returned the wrong descriptors. + let string = crate::value::js_get_string_pointer_unified(f64::from_bits( + str_handle.get_heap_word_u64(), + )) as *const crate::StringHeader; + let cstr = crate::string::js_string_char_at(string, index as i32); + let char_val = f64::from_bits(JSValue::string_ptr(cstr).bits()); + return build_data_descriptor(char_val, false, true, false); } } f64::from_bits(crate::value::TAG_UNDEFINED) diff --git a/crates/perry-runtime/src/object/field_get_set/enumeration.rs b/crates/perry-runtime/src/object/field_get_set/enumeration.rs index d5a60122ab..30098dd73d 100644 --- a/crates/perry-runtime/src/object/field_get_set/enumeration.rs +++ b/crates/perry-runtime/src/object/field_get_set/enumeration.rs @@ -151,48 +151,6 @@ pub extern "C" fn js_object_keys_value(value: f64) -> *mut ArrayHeader { } return arr; } - if crate::builtins::boxed_primitive_to_string_tag(value) == Some("String") { - if let Some((_, payload)) = crate::builtins::boxed_primitive_payload(value) { - let mut scratch = [0u8; crate::value::SHORT_STRING_MAX_LEN]; - let len = match crate::string::str_bytes_from_jsvalue(payload, &mut scratch) { - Some((ptr, blen)) if !ptr.is_null() => crate::string::compute_utf16_len(ptr, blen), - _ => 0, - }; - let arr = crate::array::js_array_alloc(len.max(1)); - for i in 0..len { - let s = i.to_string(); - let k = crate::string::js_string_from_bytes(s.as_ptr(), s.len() as u32); - crate::array::js_array_push(arr, JSValue::string_ptr(k)); - } - if jv.is_pointer() { - let ptr = jv.as_pointer::(); - let own = js_object_keys(ptr); - let own_len = crate::array::js_array_length(own); - for i in 0..own_len { - let key_val = crate::array::js_array_get(own, i); - // The wrapper's character indices are installed as REAL - // own fields at construction (install_string_wrapper_ - // indices), so they come back from `js_object_keys` too — - // skip them here or `Object.keys(Object("abc"))` lists - // every index twice. Only canonical indices below the - // string length are virtual; expando keys pass through. - let key_ptr = - (key_val.bits() & crate::value::POINTER_MASK) as *const crate::StringHeader; - if let Some(name) = - unsafe { super::super::has_own_helpers::str_from_string_header(key_ptr) } - { - if let Ok(idx) = name.parse::() { - if idx.to_string() == name && (idx as usize) < len as usize { - continue; - } - } - } - crate::array::js_array_push_f64(arr, f64::from_bits(key_val.bits())); - } - } - return arr; - } - } if let Some(addr) = crate::typedarray_props::typed_array_addr_from_value(value) { return unsafe { crate::typedarray_props::typed_array_own_property_names( @@ -1249,6 +1207,12 @@ fn js_object_keys_shape(obj: *const ObjectHeader) -> *mut ArrayHeader { } } unsafe { + if let Some(result) = super::super::string_wrapper::enumerate( + obj, + super::super::string_wrapper::Enumeration::Keys, + ) { + return result; + } if (*obj).class_id == NATIVE_MODULE_CLASS_ID { // Relocated to native_module.rs::vt_own_keys_array so the // module key tables are reachable only through the vtable @@ -1547,6 +1511,12 @@ fn js_object_values_shape(obj: *const ObjectHeader) -> *mut ArrayHeader { return crate::array::js_array_alloc(0); } unsafe { + if let Some(result) = super::super::string_wrapper::enumerate( + obj, + super::super::string_wrapper::Enumeration::Values, + ) { + return result; + } if (*obj).class_id == NATIVE_MODULE_CLASS_ID { if let Some(result) = native_module_enum(obj, MapSetEnum::Values) { return result; @@ -1758,6 +1728,12 @@ fn js_object_entries_shape(obj: *const ObjectHeader) -> *mut ArrayHeader { return crate::array::js_array_alloc(0); } unsafe { + if let Some(result) = super::super::string_wrapper::enumerate( + obj, + super::super::string_wrapper::Enumeration::Entries, + ) { + return result; + } if (*obj).class_id == NATIVE_MODULE_CLASS_ID { if let Some(result) = native_module_enum(obj, MapSetEnum::Entries) { return result; diff --git a/crates/perry-runtime/src/object/field_set_by_name/tail.rs b/crates/perry-runtime/src/object/field_set_by_name/tail.rs index ee6a361400..632bccf67a 100644 --- a/crates/perry-runtime/src/object/field_set_by_name/tail.rs +++ b/crates/perry-runtime/src/object/field_set_by_name/tail.rs @@ -234,6 +234,11 @@ pub(crate) fn set_field_by_name_object_tail( } } + if super::super::string_wrapper::has_index_key(obj as usize, key) { + let name = key_to_str_for_diag(key); + crate::error::throw_immutable_write(0, &name); + } + // Resolve the interned key EARLY (hoisted from below the interception // vet): the store-plan cache and the shape-transition cache both key // on interned pointer identity. If the key is already interned diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index c7b48fc87d..87dbfafc47 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -103,6 +103,7 @@ mod instanceof; mod live_slots; mod null_stub; mod side_table_roots; +mod string_wrapper; pub(crate) use live_slots::set_object_live_slot_count; pub use live_slots::{ js_object_live_slot_count, object_live_slot_count, perry_object_header_abi_revision, @@ -662,21 +663,6 @@ fn shape_cache_get_with_id(shape_id: u32) -> (*mut ArrayHeader, u32) { .unwrap_or((std::ptr::null_mut(), 0)) } -/// Rule 1 of `gc/young_log.rs` for the shape cache: log `shape_id` BEFORE the -/// entry naming `keys_array` becomes findable. -/// -/// Every writer of the cache — the production `shape_cache_insert` and the -/// `#[cfg(test)]` seed seam — arms through this one function. A seam that -/// re-implements the predicate is the failure mode this exists to prevent: -/// the tests then validate an arming rule that is not the one that ships, and -/// deleting the production arm site stays green. -#[inline] -pub(super) fn arm_shape_cache_young(shape_id: u32, keys_array: *mut ArrayHeader) { - if crate::gc::young_log::addr_is_minor_relevant(keys_array as usize) { - SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().note(shape_id)); - } -} - /// Insert a keys_array into the cache. Updates the inline slot /// (evicting any prior entry there) and also writes to the overflow /// map so misses on the inline cache still find the value. @@ -706,9 +692,6 @@ fn shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) { }; let st = crate::state::state(); let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); - // #9754 rule 1: log the id BEFORE the entry is published when the keys - // array can matter to a minor. - arm_shape_cache_young(shape_id, keys_array); unsafe { // GC_STORE_AUDIT(ROOT): shape_inline_cache entries are scanned by scan_shape_cache_roots_mut. let entry = &mut (*st.object_hot.shape_inline_cache.get())[slot]; @@ -822,14 +805,9 @@ crate::perry_thread_local! { /// `scan_transition_cache_roots_mut` visits only these. static TRANSITION_CACHE_YOUNG: RefCell> = const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; - /// #9754: shape-cache ids (inline slot and overflow key alike) whose keys - /// array may still be acted on by a minor. - static SHAPE_CACHE_YOUNG: RefCell> = - const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; } const TRANSITION_CACHE_YOUNG_LOG_NAME: &str = "object.transition_cache"; -const SHAPE_CACHE_YOUNG_LOG_NAME: &str = "object.shape_cache"; /// Is a transition-cache entry still something a minor can act on? #[inline] @@ -1334,7 +1312,6 @@ pub(crate) fn test_shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeade pub(crate) fn test_seed_shape_cache_root(shape_id: u32, keys_array: *mut ArrayHeader) { let st = crate::state::state(); let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); - arm_shape_cache_young(shape_id, keys_array); unsafe { // GC_STORE_AUDIT(ROOT): test seed mirrors shape_inline_cache roots scanned by scan_shape_cache_roots_mut. let entry = &mut (*st.object_hot.shape_inline_cache.get())[slot]; diff --git a/crates/perry-runtime/src/object/native_call_method.rs b/crates/perry-runtime/src/object/native_call_method.rs index e5044e8eb5..f05a2285a6 100644 --- a/crates/perry-runtime/src/object/native_call_method.rs +++ b/crates/perry-runtime/src/object/native_call_method.rs @@ -335,6 +335,20 @@ unsafe fn call_primitive_closure_value( Some(result) } +/// UTF-16 length of a string receiver, 0 for every other primitive — the +/// number of own index properties its `ToObject` wrapper would materialise. +unsafe fn primitive_receiver_utf16_len(receiver: f64) -> u64 { + let jsval = JSValue::from_bits(receiver.to_bits()); + if !jsval.is_any_string() { + return 0; + } + let ptr = crate::value::js_get_string_pointer_unified(receiver) as *const crate::StringHeader; + if ptr.is_null() { + return 0; + } + crate::string::js_string_length(ptr) as u64 +} + unsafe fn call_primitive_builtin_prototype_method( receiver: f64, builtin_name: &[u8], @@ -342,6 +356,12 @@ unsafe fn call_primitive_builtin_prototype_method( args_ptr: *const f64, args_len: usize, ) -> Option { + // #9761 attribution: this is the fork where an unrecognised primitive + // method name turns into a `globalThis` lookup plus, for a sloppy callee, + // a `ToObject` wrapper whose own index properties are O(receiver length). + crate::gc::diag_primitive_dispatch(builtin_name, method_name, unsafe { + primitive_receiver_utf16_len(receiver) + }); let ctor = crate::object::js_get_global_this_builtin_value(builtin_name.as_ptr(), builtin_name.len()); let ctor_value = JSValue::from_bits(ctor.to_bits()); @@ -374,7 +394,9 @@ unsafe fn call_primitive_builtin_prototype_method( if let Some(value) = builtin_proto_accessor_method(proto_ptr, method_name, receiver) { return call_primitive_closure_value(receiver, value, args_ptr, args_len); } - let key = crate::string::js_string_from_bytes(method_name.as_ptr(), method_name.len() as u32); + // A method name is a literal at the call site; the canonical interned + // header is allocated once per thread instead of once per dispatch. + let key = crate::string::canonical_key(method_name.as_bytes()); let value = js_object_get_field_by_name(proto_ptr, key); call_primitive_closure_value(receiver, value, args_ptr, args_len) } diff --git a/crates/perry-runtime/src/object/object_ops/define_property.rs b/crates/perry-runtime/src/object/object_ops/define_property.rs index fc3d1e4bc6..9bf54caca2 100644 --- a/crates/perry-runtime/src/object/object_ops/define_property.rs +++ b/crates/perry-runtime/src/object/object_ops/define_property.rs @@ -1474,6 +1474,11 @@ pub extern "C" fn js_object_define_property( desc_view.as_ref(), )); } + // A compatible definition of an immutable virtual index is a no-op. + // The invariant check above has already rejected every actual change. + if super::super::string_wrapper::has_index_key(obj as usize, key_str) { + return obj_value; + } super::super::mark_object_dynamic_shape_unknown(obj); // Extract descriptor object if extract_obj_ptr(descriptor_value).is_null() { diff --git a/crates/perry-runtime/src/object/object_ops/keys_array.rs b/crates/perry-runtime/src/object/object_ops/keys_array.rs index 25e32e8f70..286b78aa65 100644 --- a/crates/perry-runtime/src/object/object_ops/keys_array.rs +++ b/crates/perry-runtime/src/object/object_ops/keys_array.rs @@ -348,6 +348,9 @@ pub(crate) unsafe fn own_key_present_via_index( if (*obj).class_id == super::super::native_module::NATIVE_MODULE_CLASS_ID { return None; } + if super::super::string_wrapper::has_index_key(obj as usize, key) { + return Some(true); + } let keys = crate::object::object_keys_array(obj); match crate::value::addr_class::try_read_gc_header(keys as usize) { Some(h) if h.obj_type == crate::gc::GC_TYPE_ARRAY => {} @@ -397,6 +400,9 @@ pub(crate) unsafe fn own_key_present( Some(h) if h.obj_type == crate::gc::GC_TYPE_OBJECT => {} _ => return false, } + if super::super::string_wrapper::has_index_key(obj as usize, key) { + return true; + } let keys = crate::object::object_keys_array(obj); if keys.is_null() { return false; diff --git a/crates/perry-runtime/src/object/object_ops/prototype.rs b/crates/perry-runtime/src/object/object_ops/prototype.rs index 2dc1109e76..a4f8c418d3 100644 --- a/crates/perry-runtime/src/object/object_ops/prototype.rs +++ b/crates/perry-runtime/src/object/object_ops/prototype.rs @@ -37,8 +37,14 @@ pub extern "C" fn js_get_global_this_builtin_value(name_ptr: *const u8, name_len // one of them straddles the collection. let scope = crate::gc::RuntimeHandleScope::new(); let global_handle = scope.root_nanbox_f64(js_get_global_this()); - let (key, global_this_f64) = global_handle - .across_nanbox(|| crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32)); + // #9761: this lookup used to MINT the name string on every call — the + // comment below still records why that allocation is a collection point. + // It is now the canonical interned header, so the allocation happens once + // per thread per name instead of once per lookup: on the compiled cc TUI + // this single site was 133 MB of the 990 MB a 3300-character reply + // allocates (every primitive method call asks for `globalThis.String`). + let (key, global_this_f64) = + global_handle.across_nanbox(|| crate::string::canonical_key(name.as_bytes())); let global_obj = crate::value::js_nanbox_get_pointer(global_this_f64) as *const ObjectHeader; if global_obj.is_null() { return f64::from_bits(crate::value::TAG_UNDEFINED); diff --git a/crates/perry-runtime/src/object/prototype_helpers.rs b/crates/perry-runtime/src/object/prototype_helpers.rs index 169bab42a9..f6fc1f5285 100644 --- a/crates/perry-runtime/src/object/prototype_helpers.rs +++ b/crates/perry-runtime/src/object/prototype_helpers.rs @@ -4,8 +4,7 @@ pub(crate) fn constructor_dynamic_prototype(obj: *const ObjectHeader) -> Option< if obj.is_null() { return None; } - let key = - crate::string::js_string_from_bytes(b"constructor".as_ptr(), b"constructor".len() as u32); + let key = crate::string::canonical_key(b"constructor"); let constructor = js_object_get_field_by_name_f64(obj, key); let bits = constructor.to_bits(); let top16 = bits >> 48; diff --git a/crates/perry-runtime/src/object/reflect_support.rs b/crates/perry-runtime/src/object/reflect_support.rs index fe580da06d..6d386d28e0 100644 --- a/crates/perry-runtime/src/object/reflect_support.rs +++ b/crates/perry-runtime/src/object/reflect_support.rs @@ -181,6 +181,9 @@ pub(crate) fn obj_value_has_own_key(value: f64, key: f64) -> bool { if let Some(present) = crate::process::process_env_has_field(obj, key_str) { return present; } + if super::string_wrapper::has_index_key(obj as usize, key_str) { + return true; + } // #9190 replaced the allocating per-element `js_array_get` walk with // the consult-only key index below, so no handle round-trip is needed: // there is no collection point between reading these pointers and diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index f61cf84098..3e0ebf1a01 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -45,7 +45,7 @@ pub(crate) use shapes_slot_list::{ object_shape_hole_count, publish_object_shape_holes, rekey_stable_tombstone_shape_after_squeeze, retire_owned_shape_history, shape_index_migrate_after_delete, shape_index_shift_in_place, - try_update_stable_tombstone_shape, try_update_stable_tombstone_shape_cached, SlotList, + try_update_stable_tombstone_shape, try_update_stable_tombstone_shape_cached, SlotIndex, }; use shapes_store::{ IdList, ShapeRecord, ShapeSlab, RECORD_FLAG_CACHE_CARRIER, RECORD_FLAG_CARRIED_SEEN, @@ -68,7 +68,7 @@ pub(crate) struct ShapeIndex { /// `bench_populated_delete.ts` — perry's worst object-model gap against /// node — `hash_one::<&usize>` plus `sip::Hasher::write` were **14.7% of /// self time**, second only to the lookup that performs them. - slots: crate::fast_hash::PtrHashMap, + slots: SlotIndex, } /// Immutable facts named by one ShapeId, copied out of the table. @@ -1627,12 +1627,7 @@ unsafe fn index_range(shape: &mut ShapeIndex, keys: *const ArrayHeader, key_coun let v = crate::JSValue::from_bits((*slots.add(i as usize)).to_bits()); if let Some(b) = crate::string::js_string_key_bytes(v, &mut sso) { let h = super::key_bytes_hash(b.as_ptr(), b.len()); - match shape.slots.entry(h) { - std::collections::hash_map::Entry::Occupied(mut e) => e.get_mut().push(i), - std::collections::hash_map::Entry::Vacant(e) => { - e.insert(SlotList::One(i)); - } - } + shape.slots.push(h, i); } } shape.indexed_len = key_count; @@ -1699,7 +1694,7 @@ pub(crate) unsafe fn shape_slot_lookup_verdict( inner.note_young_keys(keys_id as u64); inner.indices.entry(keys_id).or_insert(ShapeIndex { indexed_len: 0, - slots: crate::fast_hash::new_ptr_hash_map(), + slots: SlotIndex::new(), }) } }; @@ -1712,12 +1707,9 @@ pub(crate) unsafe fn shape_slot_lookup_verdict( } else { KeysIndexVerdict::Unindexed }; - let Some(candidates) = shape.slots.get(&key_hash) else { - return absent; - }; let mut sso = [0u8; crate::value::SHORT_STRING_MAX_LEN]; let (slots, slot_len) = super::keys_array_dense_slots(keys); - for &i in candidates.iter() { + for i in shape.slots.candidates(key_hash) { if (i as usize) >= slot_len || i >= key_count { continue; } @@ -1746,12 +1738,7 @@ pub(crate) fn shape_note_append( if let Some(shape) = inner.indices.get_mut(&(keys as usize)) { if shape.indexed_len + 1 == new_count { shape.indexed_len = new_count; - match shape.slots.entry(key_hash) { - std::collections::hash_map::Entry::Occupied(mut e) => e.get_mut().push(slot), - std::collections::hash_map::Entry::Vacant(e) => { - e.insert(SlotList::One(slot)); - } - } + shape.slots.push(key_hash, slot); } } } @@ -1761,12 +1748,7 @@ pub(crate) fn shape_note_append( pub(crate) fn shape_note_hit(keys: *const ArrayHeader, key_hash: u64, slot: u32) { let mut inner = crate::state::state().shapes.inner.borrow_mut(); if let Some(shape) = inner.indices.get_mut(&(keys as usize)) { - match shape.slots.entry(key_hash) { - std::collections::hash_map::Entry::Occupied(mut e) => e.get_mut().push(slot), - std::collections::hash_map::Entry::Vacant(e) => { - e.insert(SlotList::One(slot)); - } - } + shape.slots.push(key_hash, slot); } } @@ -2314,17 +2296,13 @@ pub(crate) fn shrink_shape_tables() { /// `PERRY_GC_CENSUS`: the by-id slab, the per-shape key indices, the /// exact-facts accelerator and the keys-address family index. pub(crate) fn shape_table_census() -> Vec { - use crate::gc::census::{hash_table_bytes, map_bytes}; + use crate::gc::census::map_bytes; let table = &crate::state::state().shapes; let inner = table.inner.borrow(); let slab = table.slab(); let mut rows = Vec::new(); rows.push(("shapes.descriptors", slab.len(), slab.estimated_bytes())); - let index_inner: usize = inner - .indices - .values() - .map(|ix| hash_table_bytes(ix.slots.capacity(), std::mem::size_of::<(u64, SlotList)>())) - .sum(); + let index_inner: usize = inner.indices.values().map(|ix| ix.slots.heap_bytes()).sum(); rows.push(( "shapes.indices", inner.indices.len(), diff --git a/crates/perry-runtime/src/object/shapes_slot_list.rs b/crates/perry-runtime/src/object/shapes_slot_list.rs index e6f8b7b1e9..9c482d518d 100644 --- a/crates/perry-runtime/src/object/shapes_slot_list.rs +++ b/crates/perry-runtime/src/object/shapes_slot_list.rs @@ -1,81 +1,283 @@ -//! `SlotList` — the shape key index's per-hash slot list, in a sibling file. +//! `SlotIndex` — the shape key index's content-hash → slot table, in a +//! sibling file. //! //! Extracted from `shapes.rs` to keep it under the repo's 2000-line cap. -//! Also carries the two helpers that are mostly `SlotList` manipulation: +//! Also carries the two helpers that are mostly index manipulation: //! `record_shape_scan_outcome` (the shape scanner's per-descriptor //! bookkeeping) and `shape_index_migrate_after_delete`. -/// Slots sharing one content hash. +/// Content hash → candidate slots for one shape, as an open-addressing table +/// of packed `(hash tag, slot)` cells. /// -/// Almost always exactly one: the key is an FNV-1a hash of distinct property -/// names, so a bucket with two entries is a genuine hash collision. Storing -/// that common case inline removes a heap allocation PER KEY from every index -/// build — and the index is rebuilt on every populated delete, so a 500-key -/// object was making ~500 `Vec` allocations per `delete`. Allocator and page -/// churn is the dominant cost on that benchmark (`clear_page_erms` 5.6%, -/// `mi_free` 4.2%, `RawVecInner::finish_grow` 2.9%), well above the lookup -/// work itself. +/// #9754 memory. This used to be a `PtrHashMap` per shape — +/// a 33-byte hashbrown bucket (`(u64, enum { One(u32), Many(Vec) })` +/// plus its control byte) for every key, in a power-of-two table. The +/// compiled claude-code TUI holds ~34.5k of these indices, one per keys array +/// past `KEYS_INDEX_THRESHOLD`, at **2.6 KB each: 89 MB** of the process's +/// 170 MB of side tables (`PERRY_GC_CENSUS`, 2026-09-04), while the objects +/// they describe are ~40 keys wide. +/// +/// The table's only job is to answer "which slots might hold a key with this +/// hash" — every hit is then re-validated against the key BYTES +/// (`shape_slot_lookup_verdict`), so a wrong or colliding answer is a miss, +/// never a wrong property. That validation is what lets the stored hash be +/// narrow: a cell is a 16-bit tag (the top of a golden-ratio fold of the FNV-1a +/// hash) and a +/// 16-bit slot (`Narrow`, 4 bytes), widened to a 16-bit tag and a 32-bit slot +/// (`Wide`, 8 bytes) only for a shape with 65 535 or more keys. The probe +/// position is a function of the tag alone, so a cell can be re-placed from +/// its own bits when the table grows or is rebuilt after a delete. Same +/// hash, several slots (a genuine collision, or a note-hit under a stale +/// index) is just several cells with one tag on the probe chain. Load is +/// kept at or below 7/8. +/// +/// 40 keys: 64 cells × 4 B = 256 B against the 2.1 KB hashbrown table. +#[derive(Clone, Debug)] +pub(crate) struct SlotIndex { + cells: SlotCells, + len: u32, +} + #[derive(Clone, Debug)] -pub(crate) enum SlotList { - One(u32), - Many(Vec), +enum SlotCells { + /// `(tag16 << 16) | slot16`; slots up to `NARROW_MAX_SLOT`. + Narrow(Box<[u32]>), + /// `(tag16 << 32) | slot32`. + Wide(Box<[u64]>), +} + +const NARROW_EMPTY: u32 = u32::MAX; +const WIDE_EMPTY: u64 = u64::MAX; +/// Slot `0xFFFF` is never stored narrow, so `NARROW_EMPTY` is unambiguous. +const NARROW_MAX_SLOT: u32 = 0xFFFE; +const MIN_CELLS: usize = 8; + +impl Default for SlotIndex { + fn default() -> Self { + Self::new() + } } -impl SlotList { +impl SlotIndex { + pub(crate) fn new() -> Self { + Self { + cells: SlotCells::Narrow(Box::new([])), + len: 0, + } + } + #[inline] - pub(crate) fn push(&mut self, slot: u32) { - match self { - SlotList::One(existing) => { - *self = SlotList::Many(vec![*existing, slot]); - } - SlotList::Many(v) => v.push(slot), + fn capacity(&self) -> usize { + match &self.cells { + SlotCells::Narrow(cells) => cells.len(), + SlotCells::Wide(cells) => cells.len(), + } + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.len as usize + } + + /// Bytes of the cell array (`PERRY_GC_CENSUS`). + pub(crate) fn heap_bytes(&self) -> usize { + match &self.cells { + SlotCells::Narrow(cells) => cells.len() * std::mem::size_of::(), + SlotCells::Wide(cells) => cells.len() * std::mem::size_of::(), } } - /// Drop `removed` and shift every slot above it down by one. + /// The 16-bit tag of a key hash. FNV-1a's HIGH bits barely move for + /// short keys (`"a"` and `"b"` share their top 16), so fold the whole + /// word through a golden-ratio multiply first and take the top of that. #[inline] - pub(crate) fn retain_shift(&mut self, removed: u32) { - let shift = |s: u32| -> Option { - match s.cmp(&removed) { - std::cmp::Ordering::Equal => None, - std::cmp::Ordering::Less => Some(s), - std::cmp::Ordering::Greater => Some(s - 1), + fn tag_of(hash: u64) -> u32 { + (hash.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 48) as u32 + } + + /// Where a tag's probe chain starts. Below 65 536 cells the tag itself + /// indexes the table; above, its two copies cover the extra bits (two + /// tags then share a chain, which is a longer probe, never a wrong answer). + #[inline] + fn home(tag: u32, mask: usize) -> usize { + ((tag as usize) | ((tag as usize) << 16)) & mask + } + + /// Record that `slot` holds a key hashing to `hash`. A cell already + /// naming exactly this pair is left alone (a note-hit on an indexed key). + pub(crate) fn push(&mut self, hash: u64, slot: u32) { + self.insert(Self::tag_of(hash), slot); + } + + fn insert(&mut self, tag: u32, slot: u32) { + if slot > NARROW_MAX_SLOT { + self.widen(); + } + if (self.len as usize + 1) * 8 > self.capacity() * 7 { + self.grow(); + } + let mask = self.capacity() - 1; + let mut pos = Self::home(tag, mask); + match &mut self.cells { + SlotCells::Narrow(cells) => { + let cell = (tag << 16) | slot; + loop { + let existing = cells[pos]; + if existing == NARROW_EMPTY { + cells[pos] = cell; + self.len += 1; + return; + } + if existing == cell { + return; + } + pos = (pos + 1) & mask; + } } - }; - match self { - SlotList::One(slot) => match shift(*slot) { - Some(s) => *slot = s, - None => *self = SlotList::Many(Vec::new()), + SlotCells::Wide(cells) => { + let cell = (u64::from(tag) << 32) | u64::from(slot); + loop { + let existing = cells[pos]; + if existing == WIDE_EMPTY { + cells[pos] = cell; + self.len += 1; + return; + } + if existing == cell { + return; + } + pos = (pos + 1) & mask; + } + } + } + } + + /// Every slot recorded under `hash`, in probe order. Each must still be + /// validated against the key bytes by the caller. + pub(crate) fn candidates(&self, hash: u64) -> SlotCandidates<'_> { + let capacity = self.capacity(); + let tag = Self::tag_of(hash); + SlotCandidates { + index: self, + pos: if capacity == 0 { + 0 + } else { + Self::home(tag, capacity - 1) }, - SlotList::Many(v) => { - v.retain_mut(|s| match shift(*s) { - Some(n) => { - *s = n; - true + remaining: capacity, + tag, + } + } + + /// Drop the cell(s) for `removed` and shift every slot above it down by + /// one — the index of a keys array after an in-place or cloned delete. + pub(crate) fn retain_shift(&mut self, removed: u32) { + let pairs = self.drain_pairs(); + for (tag, slot) in pairs { + match slot.cmp(&removed) { + std::cmp::Ordering::Equal => {} + std::cmp::Ordering::Less => self.insert(tag, slot), + std::cmp::Ordering::Greater => self.insert(tag, slot - 1), + } + } + } + + /// Take every `(tag, slot)` pair out, leaving the cells empty at the same + /// capacity. + fn drain_pairs(&mut self) -> Vec<(u32, u32)> { + let mut pairs = Vec::with_capacity(self.len as usize); + match &mut self.cells { + SlotCells::Narrow(cells) => { + for cell in cells.iter_mut() { + if *cell != NARROW_EMPTY { + pairs.push((*cell >> 16, *cell & 0xFFFF)); + *cell = NARROW_EMPTY; + } + } + } + SlotCells::Wide(cells) => { + for cell in cells.iter_mut() { + if *cell != WIDE_EMPTY { + pairs.push(((*cell >> 32) as u32, (*cell & 0xFFFF_FFFF) as u32)); + *cell = WIDE_EMPTY; } - None => false, - }); - if v.len() == 1 { - *self = SlotList::One(v[0]); } } } + self.len = 0; + pairs } - #[inline] - pub(crate) fn is_empty(&self) -> bool { - match self { - SlotList::One(_) => false, - SlotList::Many(v) => v.is_empty(), + fn grow(&mut self) { + let new_capacity = (self.capacity() * 2).max(MIN_CELLS); + let pairs = self.drain_pairs(); + self.cells = match self.cells { + SlotCells::Narrow(_) => SlotCells::Narrow(vec![NARROW_EMPTY; new_capacity].into()), + SlotCells::Wide(_) => SlotCells::Wide(vec![WIDE_EMPTY; new_capacity].into()), + }; + for (tag, slot) in pairs { + self.insert(tag, slot); } } - #[inline] - pub(crate) fn iter(&self) -> impl Iterator { - match self { - SlotList::One(slot) => std::slice::from_ref(slot).iter(), - SlotList::Many(v) => v.iter(), + fn widen(&mut self) { + if matches!(self.cells, SlotCells::Wide(_)) { + return; + } + let capacity = self.capacity().max(MIN_CELLS); + let pairs = self.drain_pairs(); + self.cells = SlotCells::Wide(vec![WIDE_EMPTY; capacity].into()); + for (tag, slot) in pairs { + self.insert(tag, slot); + } + } +} + +/// The probe chain of [`SlotIndex::candidates`]. +pub(crate) struct SlotCandidates<'a> { + index: &'a SlotIndex, + pos: usize, + remaining: usize, + tag: u32, +} + +impl Iterator for SlotCandidates<'_> { + type Item = u32; + + fn next(&mut self) -> Option { + let capacity = self.index.capacity(); + if capacity == 0 { + return None; + } + let mask = capacity - 1; + while self.remaining != 0 { + self.remaining -= 1; + let pos = self.pos; + self.pos = (pos + 1) & mask; + match &self.index.cells { + SlotCells::Narrow(cells) => { + let cell = cells[pos]; + if cell == NARROW_EMPTY { + self.remaining = 0; + return None; + } + if cell >> 16 == self.tag { + return Some(cell & 0xFFFF); + } + } + SlotCells::Wide(cells) => { + let cell = cells[pos]; + if cell == WIDE_EMPTY { + self.remaining = 0; + return None; + } + if (cell >> 32) as u32 == self.tag { + return Some((cell & 0xFFFF_FFFF) as u32); + } + } + } } + None } } @@ -108,10 +310,7 @@ pub(crate) fn shape_index_shift_in_place( inner.indices.remove(&keys_id); return false; } - index.slots.retain(|_, list| { - list.retain_shift(removed_slot); - !list.is_empty() - }); + index.slots.retain_shift(removed_slot); index.indexed_len = old_key_count - 1; true } @@ -165,10 +364,7 @@ pub(crate) fn shape_index_migrate_after_delete( // misaligned. Dropping it preserves the previous behaviour exactly. return false; } - index.slots.retain(|_, list| { - list.retain_shift(removed_slot); - !list.is_empty() - }); + index.slots.retain_shift(removed_slot); index.indexed_len = old_key_count - 1; inner.note_young_keys(new_keys_id as u64); inner.indices.insert(new_keys_id, index); @@ -692,3 +888,79 @@ mod tests { } } } + +#[cfg(test)] +mod slot_index_tests { + use super::SlotIndex; + + fn fnv(bytes: &[u8]) -> u64 { + crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()) + } + + #[test] + fn every_pushed_pair_is_a_candidate_and_nothing_else_is() { + let mut index = SlotIndex::new(); + let names: Vec = (0..3000).map(|i| format!("key_{i}")).collect(); + for (slot, name) in names.iter().enumerate() { + index.push(fnv(name.as_bytes()), slot as u32); + } + assert_eq!(index.len(), 3000); + for (slot, name) in names.iter().enumerate() { + let found: Vec = index.candidates(fnv(name.as_bytes())).collect(); + assert!(found.contains(&(slot as u32)), "{name} missing: {found:?}"); + } + let absent: Vec = index.candidates(fnv(b"never_inserted")).collect(); + assert!( + absent.len() <= 2, + "a narrow tag should almost never alias: {absent:?}" + ); + assert!( + index.heap_bytes() <= 4096 * 4, + "3000 keys must fit 4096 narrow cells" + ); + } + + #[test] + fn a_repeated_note_hit_does_not_grow_the_table() { + let mut index = SlotIndex::new(); + let hash = fnv(b"hit"); + for _ in 0..100 { + index.push(hash, 7); + } + assert_eq!(index.len(), 1); + assert_eq!(index.candidates(hash).collect::>(), vec![7]); + } + + #[test] + fn retain_shift_drops_the_removed_slot_and_shifts_the_rest() { + let mut index = SlotIndex::new(); + let names: Vec = (0..50).map(|i| format!("k{i}")).collect(); + for (slot, name) in names.iter().enumerate() { + index.push(fnv(name.as_bytes()), slot as u32); + } + index.retain_shift(10); + assert_eq!(index.len(), 49); + assert!(index.candidates(fnv(b"k10")).next().is_none()); + for (slot, name) in names.iter().enumerate() { + if slot == 10 { + continue; + } + let expected = if slot > 10 { slot - 1 } else { slot } as u32; + let found: Vec = index.candidates(fnv(name.as_bytes())).collect(); + assert_eq!(found, vec![expected], "{name}"); + } + } + + #[test] + fn a_slot_past_the_narrow_range_widens_the_table() { + let mut index = SlotIndex::new(); + index.push(fnv(b"a"), 3); + index.push(fnv(b"b"), 70_000); + assert_eq!(index.candidates(fnv(b"a")).collect::>(), vec![3]); + assert_eq!( + index.candidates(fnv(b"b")).collect::>(), + vec![70_000] + ); + assert!(index.heap_bytes() >= 8 * 8, "wide cells are 8 bytes"); + } +} diff --git a/crates/perry-runtime/src/object/shapes_test_support.rs b/crates/perry-runtime/src/object/shapes_test_support.rs index 29977c46eb..6cb974c98c 100644 --- a/crates/perry-runtime/src/object/shapes_test_support.rs +++ b/crates/perry-runtime/src/object/shapes_test_support.rs @@ -138,18 +138,16 @@ pub(crate) fn test_shape_ids_for_keys(keys_id: usize) -> Vec { #[cfg(test)] pub(crate) fn test_seed_shape_entry(keys_id: usize) { - crate::state::state() - .shapes - .inner - .borrow_mut() - .indices - .insert( - keys_id, - ShapeIndex { - indexed_len: 0, - slots: crate::fast_hash::new_ptr_hash_map(), - }, - ); + let mut inner = crate::state::state().shapes.inner.borrow_mut(); + inner.note_young_keys(keys_id as u64); + inner.indices.insert( + keys_id, + ShapeIndex { + indexed_len: 0, + slots: SlotIndex::new(), + }, + ); + drop(inner); let _ = shape_descriptor_ensure(keys_id as *const ArrayHeader, 0, 0) .expect("test shape id range unexpectedly exhausted"); } diff --git a/crates/perry-runtime/src/object/side_table_roots.rs b/crates/perry-runtime/src/object/side_table_roots.rs index fdaa3262c3..9bc56840ba 100644 --- a/crates/perry-runtime/src/object/side_table_roots.rs +++ b/crates/perry-runtime/src/object/side_table_roots.rs @@ -295,74 +295,24 @@ pub fn scan_shape_cache_roots(mark: &mut dyn FnMut(f64)) { scan_shape_cache_roots_mut(&mut visitor); } +/// #9754 measured this table and left it alone: a young-entry log here skipped +/// NOTHING on the compiled claude-code TUI (0.0 % of 3.85 M entry visits over +/// 107 collections) and cost 35 % MORE than this plain walk, because the +/// canonical keys arrays live in the LONGLIVED arena, which +/// `addr_is_minor_relevant` must answer `true` for, so no entry ever leaves +/// the log. See the four tables that do skip 75-93 % in `gc/young_log.rs`. pub fn scan_shape_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - use crate::gc::young_log::addr_is_minor_relevant; let st = crate::state::state(); - // The inline array is 256 fixed slots: always walked. The overflow map - // holds every shape id ever cached; #9754: a minor-scoped pass visits - // only the young-logged ids there, a full pass rebuilds the log. - let entries = unsafe { &mut *st.object_hot.shape_inline_cache.get() }; - for entry in entries.iter_mut() { - visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array); - } - let mut cache = st.object_hot.shape_cache_overflow.borrow_mut(); - let table_len = cache.len() as u64; - if visitor.young_scope() { - #[cfg(debug_assertions)] - { - let relevant: Vec = cache - .iter() - .filter(|(_, (arr_ptr, _))| addr_is_minor_relevant(*arr_ptr as usize)) - .map(|(&id, _)| id) - .collect(); - SHAPE_CACHE_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(SHAPE_CACHE_YOUNG_LOG_NAME, &relevant) - }); + { + let entries = unsafe { &mut *st.object_hot.shape_inline_cache.get() }; + for entry in entries.iter_mut() { + visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array); } - let batch = SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); - let logged = batch.len() as u64; - let mut kept = SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().take_spare()); - for id in batch { - if let Some((arr_ptr, _)) = cache.get_mut(&id) { - visitor.visit_raw_mut_ptr_slot(arr_ptr); - if addr_is_minor_relevant(*arr_ptr as usize) { - kept.push(id); - } - } - } - let kept_len = kept.len() as u64; - SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().extend(kept)); - crate::gc::young_log::note_walk( - SHAPE_CACHE_YOUNG_LOG_NAME, - crate::gc::young_log::YoungLogWalk { - partial: true, - logged, - visited: logged, - kept: kept_len, - table_len, - }, - ); - return; } - let _ = SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); - let mut kept = Vec::new(); - for (&id, (arr_ptr, _runtime_shape_id)) in cache.iter_mut() { - visitor.visit_raw_mut_ptr_slot(arr_ptr); - if addr_is_minor_relevant(*arr_ptr as usize) { - kept.push(id); + { + let mut cache = st.object_hot.shape_cache_overflow.borrow_mut(); + for (arr_ptr, _runtime_shape_id) in cache.values_mut() { + visitor.visit_raw_mut_ptr_slot(arr_ptr); } } - let kept_len = kept.len() as u64; - SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().extend(kept)); - crate::gc::young_log::note_walk( - SHAPE_CACHE_YOUNG_LOG_NAME, - crate::gc::young_log::YoungLogWalk { - partial: false, - logged: table_len, - visited: table_len, - kept: kept_len, - table_len, - }, - ); } diff --git a/crates/perry-runtime/src/object/string_wrapper.rs b/crates/perry-runtime/src/object/string_wrapper.rs new file mode 100644 index 0000000000..c827102e19 --- /dev/null +++ b/crates/perry-runtime/src/object/string_wrapper.rs @@ -0,0 +1,227 @@ +//! Virtual character indices for String exotic objects (#9810). +//! +//! Boxing stores the primitive and the fixed `length` property. Indices never +//! enter shapes or descriptor_state: a call with an unused string receiver +//! must not allocate one field, descriptor, and character per code unit. + +use super::{ObjectHeader, PropertyAttrs}; +use crate::{ArrayHeader, JSValue, StringHeader}; + +/// Consult only: no allocation, string coercion, or user code. Heap strings +/// already cache their UTF-16 length, so probes are independent of that length. +pub(super) fn length(owner: usize) -> Option { + unsafe { + let header = crate::value::addr_class::try_read_gc_header(owner)?; + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || (*(owner as *const ObjectHeader)).class_id != 0xFFFF_00D1 + { + return None; + } + let (_, payload) = crate::builtins::boxed_primitive_payload( + crate::value::js_nanbox_pointer(owner as i64), + )?; + let value = JSValue::from_bits(payload.to_bits()); + if value.is_string() { + let ptr = (value.bits() & crate::value::POINTER_MASK) as *const StringHeader; + return Some(crate::string::js_string_length(ptr)); + } + let mut scratch = [0; crate::value::SHORT_STRING_MAX_LEN]; + let (ptr, len) = crate::string::str_bytes_from_jsvalue(payload, &mut scratch)?; + Some(crate::string::compute_utf16_len(ptr, len)) + } +} + +pub(super) fn has_index(owner: usize, name: &str) -> bool { + // Reject ordinary property names before probing the receiver metadata. + let Some(index) = super::canonical_array_index(name) else { + return false; + }; + length(owner).is_some_and(|len| index < len) +} + +pub(super) unsafe fn has_index_key(owner: usize, key: *const StringHeader) -> bool { + crate::string::header_str_checked(key).is_some_and(|name| has_index(owner, name)) +} + +pub(super) enum Enumeration { + Keys, + Values, + Entries, +} + +unsafe fn is_enumerable(obj: *const ObjectHeader, key: *const StringHeader) -> bool { + super::own_key_present(obj as *mut ObjectHeader, key) + && crate::string::header_str_checked(key).is_some_and(|name| { + super::get_property_attrs(obj as usize, name) + .unwrap_or(PropertyAttrs::new(true, true, true)) + .enumerable() + }) +} + +/// EnumerableOwnProperties over virtual indices and ordinary expando keys. +/// Snapshot once, then recheck ownership/enumerability before each value read: +/// an expando getter can delete or hide a later property and can trigger GC. +pub(super) unsafe fn enumerate( + obj: *const ObjectHeader, + kind: Enumeration, +) -> Option<*mut ArrayHeader> { + length(obj as usize)?; + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_h = scope.root_raw_const_ptr(obj); + let names = obj_h.with_const_ptr(|obj: *const ObjectHeader| { + super::js_object_get_own_property_names(crate::value::js_nanbox_pointer(obj as i64)) + }); + let names_h = scope.root_nanbox_f64(names); + let count = crate::array::js_array_length(crate::value::js_nanbox_get_pointer( + names_h.get_nanbox_f64(), + ) as *const ArrayHeader); + let result = crate::array::js_array_alloc(count); + let result_h = scope.root_raw_mut_ptr(result); + for i in 0..count { + let iter_scope = crate::gc::RuntimeHandleScope::new(); + let key = crate::array::js_array_get( + crate::value::js_nanbox_get_pointer(names_h.get_nanbox_f64()) as *const ArrayHeader, + i, + ); + let key_h = iter_scope.root_nanbox_u64(key.bits()); + let key_ptr = crate::builtins::js_string_coerce(key_h.get_nanbox_f64()); + let key_ptr_h = iter_scope.root_string_ptr(key_ptr); + let enumerable = + obj_h.with_const_ptr(|obj| key_ptr_h.with_const_ptr(|key| is_enumerable(obj, key))); + if !enumerable { + continue; + } + let output = match kind { + Enumeration::Keys => key_h.get_nanbox_u64(), + Enumeration::Values | Enumeration::Entries => { + let value = obj_h.with_const_ptr(|obj| { + key_ptr_h.with_const_ptr(|key| super::js_object_get_field_by_name(obj, key)) + }); + if matches!(kind, Enumeration::Values) { + value.bits() + } else { + let value_h = iter_scope.root_nanbox_u64(value.bits()); + let pair = crate::array::js_array_alloc(2); + let pair_h = iter_scope.root_raw_mut_ptr(pair); + pair_h.with_mut_ptr(|pair| { + crate::array::js_array_push_f64(pair, key_h.get_nanbox_f64()) + }); + pair_h.with_mut_ptr(|pair| { + crate::array::js_array_push_f64(pair, value_h.get_nanbox_f64()) + }); + pair_h.with_mut_ptr(|pair: *mut ArrayHeader| JSValue::array_ptr(pair).bits()) + } + } + }; + result_h + .with_mut_ptr(|result| crate::array::js_array_push(result, JSValue::from_bits(output))); + } + Some(result_h.with_mut_ptr(|result| result)) +} + +/// The ordinary rest helper copies physical slots. String indices have no +/// slots, so read the included enumerable properties by name instead. +pub(super) unsafe fn rest( + source: *const ObjectHeader, + excluded: *const ArrayHeader, +) -> *mut ObjectHeader { + let scope = crate::gc::RuntimeHandleScope::new(); + let source_h = scope.root_raw_const_ptr(source); + let excluded_h = scope.root_raw_const_ptr(excluded); + let keys = enumerate(source, Enumeration::Keys).unwrap(); + let keys_h = scope.root_raw_const_ptr(keys); + let result = super::js_object_alloc(0, 0); + let result_h = scope.root_raw_mut_ptr(result); + let count = keys_h.with_const_ptr(|keys| crate::array::js_array_length(keys)); + for i in 0..count { + let iter_scope = crate::gc::RuntimeHandleScope::new(); + let key = keys_h.with_const_ptr(|keys| crate::array::js_array_get(keys, i)); + let key_h = iter_scope.root_nanbox_u64(key.bits()); + let mut scratch = [0; crate::value::SHORT_STRING_MAX_LEN]; + let bytes = crate::string::js_string_key_bytes(key, &mut scratch).unwrap(); + let skip = excluded_h.with_const_ptr(|excluded: *const ArrayHeader| { + !excluded.is_null() + && (0..crate::array::js_array_length(excluded)).any(|j| { + crate::string::js_string_key_matches_bytes( + crate::array::js_array_get(excluded, j), + bytes, + ) + }) + }); + if skip { + continue; + } + let key_ptr = crate::builtins::js_string_coerce(key_h.get_nanbox_f64()); + let key_ptr_h = iter_scope.root_string_ptr(key_ptr); + if !source_h + .with_const_ptr(|source| key_ptr_h.with_const_ptr(|key| is_enumerable(source, key))) + { + continue; + } + let value = source_h.with_const_ptr(|source| { + key_ptr_h.with_const_ptr(|key| super::js_object_get_field_by_name(source, key)) + }); + result_h.with_mut_ptr(|result| { + key_ptr_h.with_const_ptr(|key| { + super::js_object_set_field_by_name(result, key, f64::from_bits(value.bits())) + }) + }); + } + result_h.with_mut_ptr(|result| result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn boxing_and_reflection_do_not_store_character_properties() { + unsafe { + let text = "a".repeat(4096); + let string = crate::string::js_string_from_bytes(text.as_ptr(), text.len() as u32); + let boxed = crate::builtins::js_boxed_string_new( + crate::value::js_nanbox_string(string as i64), + 1, + ); + let scope = crate::gc::RuntimeHandleScope::new(); + let boxed_h = scope.root_nanbox_f64(boxed); + let check_storage = || { + let owner = crate::value::js_nanbox_get_pointer(boxed_h.get_nanbox_f64()) as usize; + assert_eq!(length(owner), Some(4096)); + let physical_keys = crate::object::object_keys_array(owner as *const ObjectHeader); + assert_eq!( + crate::array::js_array_length(physical_keys), + 1, + "only length is stored" + ); + assert_eq!( + crate::state::state() + .descriptors + .property_descriptors + .borrow() + .keys() + .filter(|(ptr, _)| *ptr == owner) + .count(), + 1, + "indices must not populate descriptor_state", + ); + assert!(has_index(owner, "4095")); + assert!(!has_index(owner, "4096")); + assert!(!has_index(owner, "01")); + let attrs = crate::object::get_property_attrs(owner, "4095").unwrap(); + assert!(!attrs.writable() && attrs.enumerable() && !attrs.configurable()); + }; + check_storage(); + let keys = crate::object::js_object_keys_value(boxed_h.get_nanbox_f64()); + assert_eq!(crate::array::js_array_length(keys), 4096); + check_storage(); + let key = crate::string::js_string_from_bytes(b"4095".as_ptr(), 4); + let descriptor = crate::object::js_object_get_own_property_descriptor( + boxed_h.get_nanbox_f64(), + crate::value::js_nanbox_string(key as i64), + ); + assert!(!JSValue::from_bits(descriptor.to_bits()).is_undefined()); + check_storage(); + } + } +} diff --git a/crates/perry-runtime/src/object/test_root_accessors.rs b/crates/perry-runtime/src/object/test_root_accessors.rs index 094138aa55..d0908e2f9b 100644 --- a/crates/perry-runtime/src/object/test_root_accessors.rs +++ b/crates/perry-runtime/src/object/test_root_accessors.rs @@ -58,7 +58,6 @@ pub(crate) fn test_transition_cache_root() -> usize { #[cfg(test)] pub(crate) fn test_clear_transition_cache_root() { super::TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().clear()); - super::SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().clear()); with_transition_cache(|t| unsafe { for i in 0..TRANSITION_CACHE_SIZE { // GC_STORE_AUDIT(ROOT): test clear writes non-pointer sentinels into scanned TRANSITION_CACHE_GLOBAL roots. diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index b6787e3c51..3be652b229 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -42,6 +42,17 @@ mod exec_array; #[cfg(feature = "regex-engine")] mod flags; #[cfg(feature = "regex-engine")] +mod program_key; +#[cfg(feature = "regex-engine")] +mod replace_expand_fancy; +#[cfg(feature = "regex-engine")] +pub(crate) use program_key::{ProgramKey, NEVER_MATCH_PATTERN}; +#[cfg(feature = "regex-engine")] +pub use replace_expand_fancy::{ + js_string_replace_all_regex, js_string_replace_regex, js_string_search_regex, + js_string_split_regex, js_string_split_regex_n, +}; +#[cfg(feature = "regex-engine")] mod global_guards; #[cfg(feature = "regex-engine")] mod global_scan; @@ -365,14 +376,14 @@ pub(crate) unsafe fn regex_gc_slot_ptrs(re: *mut RegExpHeader) -> (*mut u64, usi #[cfg(feature = "regex-engine")] crate::perry_thread_local! { /// Cache of compiled regex objects, keyed by (pattern, flags). - static REGEX_CACHE: RefCell>> = RefCell::new(HashMap::new()); + static REGEX_CACHE: RefCell>> = RefCell::new(HashMap::new()); /// Fancy-regex fallback cache for patterns with lookbehind/lookahead. - static FANCY_CACHE: RefCell>> = RefCell::new(HashMap::new()); + static FANCY_CACHE: RefCell>> = RefCell::new(HashMap::new()); /// ECMAScript backtracking matchers for quantified capture groups. These /// are the patterns where `regex`/`fancy-regex` cannot reproduce /// `RepeatMatcher` capture reset and nullable-iteration semantics (#5897). - static REPEAT_MATCHER_CACHE: RefCell>> = RefCell::new(HashMap::new()); + static REPEAT_MATCHER_CACHE: RefCell>> = RefCell::new(HashMap::new()); /// `(pattern, flags)` pairs that have already cleared construction-time /// validation. Validity is a pure function of the pair, so the answer is @@ -466,16 +477,6 @@ pub(crate) fn build_fancy_regex(pattern: &str) -> Result(cache: &mut HashMap<(String, String), V>) { +fn evict_regex_cache_if_full(cache: &mut HashMap) { if cache.len() >= REGEX_CACHE_MAX_ENTRIES { cache.clear(); if crate::hot_diag::regex_on() { @@ -518,28 +519,65 @@ fn evict_regex_cache_if_full(cache: &mut HashMap<(String, String), V>) { /// Returns `false` when BOTH engines reject it — nothing is cached and the /// caller decides whether that is a SyntaxError (see `js_regexp_new`'s /// bare-pattern fallback for the flag-prefix size edge). +/// One shared never-match program per thread. +/// +/// Only used by the `PERRY_REGEX_ENGINE=regress` measurement path, where every +/// pattern needs a value in `regex_ptr` (the built/not-built flag) but no NFA: +/// building a fresh one per pattern would be exactly the compile cost the +/// experiment exists to remove from the measurement. +#[cfg(feature = "regex-engine")] +fn shared_never_match_program() -> Arc { + crate::perry_thread_local! { + static NEVER_MATCH: RefCell>> = const { RefCell::new(None) }; + } + NEVER_MATCH.with(|slot| { + slot.borrow_mut() + .get_or_insert_with(|| Arc::new(Regex::new(NEVER_MATCH_SOURCE).unwrap())) + .clone() + }) +} + #[cfg(feature = "regex-engine")] -fn compile_and_cache_regex_checked(pattern: &str, flags: &str) -> bool { +fn compile_and_cache_regex_checked(pattern: &Arc, flags: &Arc) -> bool { let already = REGEX_CACHE.with(|cache| { cache .borrow() - .contains_key(&(pattern.to_string(), flags.to_string())) + .contains_key(&(pattern.clone(), flags.clone())) }); if already { return true; } - if let Some(repeat_matcher) = repeat_matcher::compile(pattern, flags) { + let regress_covers = if let Some(repeat_matcher) = repeat_matcher::compile(pattern, flags) { if crate::hot_diag::regex_on() { crate::hot_diag::regex_with(|d| d.compiles_repeat += 1); } REPEAT_MATCHER_CACHE.with(|cache| { + let mut cache = cache.borrow_mut(); + evict_regex_cache_if_full(&mut cache); + cache.insert((pattern.clone(), flags.clone()), Arc::new(repeat_matcher)); + }); + true + } else { + false + }; + // `PERRY_REGEX_ENGINE=regress` (measurement only — see + // `repeat_matcher::regress_first`): the ECMAScript backtracker is the + // primary engine, so stop here. Every exec-family entry point consults the + // repeat matcher first, and the shared never-match placeholder gives the + // header's `regex_ptr` built-flag a value WITHOUT building an NFA — which + // is the whole point of the experiment (the linear engine's program is + // ~12.5 KB median against regress's 512 B, measured over 4,463 literals + // from seven real bundles). + if regress_covers && repeat_matcher::regress_first() { + REGEX_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); evict_regex_cache_if_full(&mut cache); cache.insert( - (pattern.to_string(), flags.to_string()), - Arc::new(repeat_matcher), + (pattern.clone(), flags.clone()), + shared_never_match_program(), ); }); + return true; } // Translate JS regex to Rust-compatible pattern, with the inline mode // prefix the flags imply. Shared with `lazy::std_engine_syntax_ok` so the @@ -561,10 +599,7 @@ fn compile_and_cache_regex_checked(pattern: &str, flags: &str) -> bool { } let mut fc = fc.borrow_mut(); evict_regex_cache_if_full(&mut fc); - fc.insert( - (pattern.to_string(), flags.to_string()), - std::sync::Arc::new(fre), - ); + fc.insert((pattern.clone(), flags.clone()), std::sync::Arc::new(fre)); true } else { false @@ -582,17 +617,17 @@ fn compile_and_cache_regex_checked(pattern: &str, flags: &str) -> bool { REGEX_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); evict_regex_cache_if_full(&mut cache); - cache.insert((pattern.to_string(), flags.to_string()), Arc::new(regex)); + cache.insert((pattern.clone(), flags.clone()), Arc::new(regex)); }); true } #[cfg(feature = "regex-engine")] -fn get_or_compile_regex(pattern: &str, flags: &str) -> Arc { +fn get_or_compile_regex(pattern: &Arc, flags: &Arc) -> Arc { let hit = REGEX_CACHE.with(|cache| { cache .borrow() - .get(&(pattern.to_string(), flags.to_string())) + .get(&(pattern.clone(), flags.clone())) .cloned() }); if let Some(re) = hit { @@ -601,14 +636,14 @@ fn get_or_compile_regex(pattern: &str, flags: &str) -> Arc { let _ = compile_and_cache_regex_checked(pattern, flags); REGEX_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); - if let Some(re) = cache.get(&(pattern.to_string(), flags.to_string())) { + if let Some(re) = cache.get(&(pattern.clone(), flags.clone())) { return re.clone(); } // Both engines rejected it (validation normally throws before this // point) — keep the historical behavior: cache + return never-match. let arc = Arc::new(Regex::new(NEVER_MATCH_PATTERN).unwrap()); evict_regex_cache_if_full(&mut cache); - cache.insert((pattern.to_string(), flags.to_string()), arc.clone()); + cache.insert((pattern.clone(), flags.clone()), arc.clone()); arc }) } @@ -943,7 +978,14 @@ pub extern "C" fn js_regexp_new( // SyntaxError decision and populates the caches for the fancy // fallback. if !lazy::std_engine_syntax_ok(pattern_str, flags_str) - && !compile_and_cache_regex_checked(pattern_str, flags_str) + // Cold: the linear engine's parser refused, so only a BUILD + // can tell a fancy-regex pattern from a SyntaxError. + // Materialising the `Arc` key happens once per distinct + // pattern that needs the fallback, not per object. + && !compile_and_cache_regex_checked( + &Arc::from(pattern_str), + &Arc::from(flags_str), + ) { // Preserve the historical edge: validation used to test the // BARE translated pattern (no `(?ims)` prefix). A pattern that @@ -1278,7 +1320,7 @@ pub extern "C" fn js_regexp_test(re: *const RegExpHeader, s: *const StringHeader }; } - if let Some(repeat_matcher) = lookup_repeat_matcher(re) { + if let Some(repeat_matcher) = lookup_repeat_matcher_for(re, str_data, 0) { return if repeat_matcher.regex.find(str_data).is_some() { 1 } else { @@ -1359,12 +1401,87 @@ pub(crate) fn lookup_fancy_regex(re: *const RegExpHeader) -> Option bool { + unsafe { + let program = (*re).regex_ptr; + if program.is_null() { + return false; + } + let program: &Regex = &*program; + if program.as_str() == NEVER_MATCH_SOURCE { + // The `regex` crate refused this pattern (lookaround / + // backreference); it has no opinion about the subject. + return false; + } + start <= subject.len() && !program.is_match_at(subject, start) + } +} + +/// The source of the never-match program `compile_and_cache_regex_checked` +/// installs for a pattern only another engine can serve. Compared by TEXT +/// rather than by `Arc` identity so this stays independent of how the +/// placeholder is allocated. +#[cfg(feature = "regex-engine")] +const NEVER_MATCH_SOURCE: &str = r"[^\s\S]"; + +/// [`lookup_repeat_matcher`] with the linear pre-check applied: `None` also +/// when the linear program proves no match at or after `start`, so the +/// backtracker is never entered on a subject that cannot match. Every +/// `&str`-subject call site uses this; the WTF-8/UTF-16 replace path, which has +/// no `&str` to hand, uses the bare lookup. +#[cfg(feature = "regex-engine")] +fn lookup_repeat_matcher_for( + re: *const RegExpHeader, + subject: &str, + start: usize, +) -> Option> { + let matcher = lookup_repeat_matcher(re)?; + if linear_rules_out_match(re, subject, start) { + return None; + } + Some(matcher) +} + /// Look up the ECMAScript-native matcher used when quantified capture groups /// make `RepeatMatcher`'s capture-reset semantics observable. #[cfg(feature = "regex-engine")] @@ -1393,7 +1510,7 @@ fn lookup_repeat_matcher( REPEAT_MATCHER_CACHE.with(|cache| { cache .borrow() - .get(&(pat.to_string(), flags_str.to_string())) + .get(&(Arc::from(pat), Arc::from(flags_str))) .cloned() }) } @@ -1411,397 +1528,6 @@ fn lookup_repeat_matcher( /// the `regex` crate can't compile (lookbehind/backreferences) still gets full /// `$1`/`$`/`$&`/`` $` ``/`$'`/`$$` substitution. #[cfg(feature = "regex-engine")] -fn expand_js_replacement_fancy( - repl: &str, - caps: &fancy_regex::Captures, - subject: &str, - has_named_groups: bool, -) -> String { - let m0 = match caps.get(0) { - Some(m) => m, - None => return String::new(), - }; - let (mstart, mend) = (m0.start(), m0.end()); - let ngroups = caps.len(); - let b = repl.as_bytes(); - let mut out = String::with_capacity(repl.len() + 16); - let mut i = 0; - while i < b.len() { - if b[i] != b'$' { - let start = i; - while i < b.len() && b[i] != b'$' { - i += 1; - } - out.push_str(&repl[start..i]); - continue; - } - if i + 1 >= b.len() { - out.push('$'); - i += 1; - continue; - } - match b[i + 1] { - b'$' => { - out.push('$'); - i += 2; - } - b'&' => { - out.push_str(&subject[mstart..mend]); - i += 2; - } - b'`' => { - out.push_str(&subject[..mstart]); - i += 2; - } - b'\'' => { - out.push_str(&subject[mend..]); - i += 2; - } - b'0'..=b'9' => { - let d1 = (b[i + 1] - b'0') as usize; - let (group, consumed) = if i + 2 < b.len() && b[i + 2].is_ascii_digit() { - let two = d1 * 10 + (b[i + 2] - b'0') as usize; - if two >= 1 && two < ngroups { - (Some(two), 2) - } else if d1 >= 1 && d1 < ngroups { - (Some(d1), 1) - } else { - (None, 0) - } - } else if d1 >= 1 && d1 < ngroups { - (Some(d1), 1) - } else { - (None, 0) - }; - match group { - Some(g) => { - if let Some(m) = caps.get(g) { - out.push_str(m.as_str()); - } - i += 1 + consumed; - } - None => { - out.push('$'); - i += 1; - } - } - } - b'<' if has_named_groups => { - if let Some(rel) = repl[i + 2..].find('>') { - let name = &repl[i + 2..i + 2 + rel]; - if let Some(m) = caps.name(name) { - out.push_str(m.as_str()); - } - i += 2 + rel + 1; - } else { - out.push('$'); - i += 1; - } - } - _ => { - out.push('$'); - i += 1; - } - } - } - out -} - -/// Fancy-regex fallback for the string-replacement (non-callback) forms of -/// `String.prototype.replace`/`replaceAll`. Drives a manual non-overlapping -/// match loop with `fancy_regex` and expands the replacement string via -/// [`expand_js_replacement_fancy`]. Used when the pattern needs -/// lookbehind/backreferences the `regex` crate can't compile. -#[cfg(feature = "regex-engine")] -unsafe fn replace_regex_str_fancy( - str_data: &str, - fre: &fancy_regex::Regex, - global: bool, - repl_str: &str, -) -> *mut StringHeader { - let has_named_groups = fre.capture_names().any(|n| n.is_some()); - // #9430: the ECMAScript scan for the global form. fancy-regex's own - // iterator drops a zero-width match that lands where the previous match - // ended, so `"a".replace(/(?<=x)?a*/g, …)`-shaped patterns lost their - // trailing (and every interior) empty replacement. - let captures_list: Vec = if global { - global_scan::fancy_captures(fre, str_data, 0) - } else { - match fre.captures(str_data) { - Ok(Some(caps)) => vec![caps], - Ok(None) | Err(_) => Vec::new(), - } - }; - let mut result = String::new(); - let mut last_end = 0usize; - for caps in &captures_list { - let full_match = caps.get(0).unwrap(); - result.push_str(&str_data[last_end..full_match.start()]); - result.push_str(&expand_js_replacement_fancy( - repl_str, - caps, - str_data, - has_named_groups, - )); - last_end = full_match.end(); - } - result.push_str(&str_data[last_end..]); - finish_replace_bytes(result.as_bytes()) -} - -/// string.replace(regex, replacement) -> string -#[cfg(feature = "regex-engine")] -#[no_mangle] -pub extern "C" fn js_string_replace_regex( - s: *const StringHeader, - re: *const RegExpHeader, - replacement: *const StringHeader, -) -> *mut StringHeader { - if !is_valid_ptr(s) { - return js_string_from_str(""); - } - - if !is_valid_regex_ptr(re) { - // If regex is null, return original string - return copy_replace_source(s); - } - if crate::hot_diag::regex_on() { - diag_note_op(re, crate::hot_diag::RegexOp::Replace); - } - - unsafe { - // The Rust string engines require scalar-value UTF-8, while Perry - // stores lone JavaScript surrogates as WTF-8. Match those subjects as - // UTF-16 code units with the ECMAScript engine and rebuild the result - // through the WTF-8-aware string builder. - if (*s).flags & crate::string::STRING_FLAG_HAS_LONE_SURROGATES != 0 { - let replacement_bytes = if is_valid_ptr(replacement) { - string_as_bytes(replacement) - } else { - b"undefined" - }; - if let Some(result) = repeat_matcher::replace_wtf8_subject( - re, - string_as_bytes(s), - replacement_bytes, - (*re).global, - ) { - return finish_replace_bytes(&result); - } - } - - let str_data = string_as_str(s); - let repl_str = if is_valid_ptr(replacement) { - string_as_str(replacement) - } else { - "undefined" - }; - - if let Some(repeat_matcher) = lookup_repeat_matcher(re) { - let result = repeat_matcher.replace(str_data, repl_str, (*re).global); - return finish_replace_bytes(result.as_bytes()); - } - - // Pattern the `regex` crate couldn't compile (lookbehind/backreferences) - // → drive the replacement through fancy-regex. Otherwise the never-match - // placeholder in `regex_ptr` would leave the input unchanged. - if let Some(fre) = lookup_fancy_regex(re) { - return replace_regex_str_fancy(str_data, &fre, (*re).global, repl_str); - } - - let regex = lazy::header_std_regex(re); - let global = (*re).global; - let has_named_groups = regex.capture_names().any(|n| n.is_some()); - - // Route through a JS-aware expander (closure form) so `$&` / `` $` `` / - // `$'` — which the regex crate's native `$` syntax doesn't support — - // are substituted per match. `$$`, `$n`, and `$` are handled too. - // #9430: `Regex::replace_all` runs the crate's own match iterator, - // whose empty-match rule is not ECMAScript's. Drive the ECMAScript - // scan and splice the replacements here instead; the non-global form - // is the same loop over a one-element list. - let captures_list: Vec = if global { - global_scan::std_captures(regex, str_data, 0) - } else { - regex.captures(str_data).into_iter().collect() - }; - if crate::hot_diag::regex_on() { - let n = captures_list.len() as u64; - crate::hot_diag::regex_with(|d| d.replace_matches += n); - } - let mut result = String::with_capacity(str_data.len()); - let mut last_end = 0usize; - for caps in &captures_list { - let full = caps.get(0).expect("capture zero is the full match"); - result.push_str(&str_data[last_end..full.start()]); - result.push_str(&expand_js_replacement( - repl_str, - caps, - str_data, - has_named_groups, - )); - last_end = full.end(); - } - result.push_str(&str_data[last_end..]); - - finish_replace_bytes(result.as_bytes()) - } -} - -/// string.replaceAll(regex, replacement) -> string -#[cfg(feature = "regex-engine")] -#[no_mangle] -pub extern "C" fn js_string_replace_all_regex( - s: *const StringHeader, - re: *const RegExpHeader, - replacement: *const StringHeader, -) -> *mut StringHeader { - if !is_valid_ptr(s) { - return js_string_from_str(""); - } - - if !is_valid_regex_ptr(re) { - return copy_replace_source(s); - } - - ensure_replace_all_regex_global(re); - js_string_replace_regex(s, re, replacement) -} - -/// Split a string by a regex delimiter -/// string.split(regex) -> string[] (array of NaN-boxed string pointers) -#[cfg(feature = "regex-engine")] -#[no_mangle] -pub extern "C" fn js_string_split_regex( - s: *const StringHeader, - re: *const RegExpHeader, -) -> *mut ArrayHeader { - js_string_split_regex_n(s, re, -1) -} - -/// string.split(regex, limit) — limit<0 means no limit, limit==0 means empty -/// (issue #567). -#[cfg(feature = "regex-engine")] -#[no_mangle] -pub extern "C" fn js_string_split_regex_n( - s: *const StringHeader, - re: *const RegExpHeader, - limit: i32, -) -> *mut ArrayHeader { - const STRING_TAG: u64 = 0x7FFF_0000_0000_0000; - const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; - - if !is_valid_ptr(s) { - return crate::array::js_array_alloc(0); - } - if limit == 0 { - return crate::array::js_array_alloc(0); - } - let str_data = string_as_str(s).to_owned(); - - if !is_valid_regex_ptr(re) { - // No regex: return array with the whole string as a single element - let arr = crate::array::js_array_alloc(1); - let scope = crate::gc::RuntimeHandleScope::new(); - let arr_handle = scope.root_raw_mut_ptr(arr); - // Allocating string + array re-read as one combinator (#7341). - let (str_ptr, arr) = - arr_handle.across_mut::(|| js_string_from_str(&str_data) as u64); - unsafe { - (*arr).length = 1; - let nanboxed = STRING_TAG | (str_ptr & POINTER_MASK); - // GC_STORE_AUDIT(BARRIERED): regex split fallback slot uses the shared array slot-store helper. - crate::array::store_array_slot(arr, 0, nanboxed); - } - return arr_handle.get_raw_mut_ptr::(); - } - - const TAG_UNDEFINED: u64 = 0x7FFC_0000_0000_0001; - unsafe { - // Each element is either a substring (`Some`) or `undefined` (`None`, - // for an unmatched capture group spliced into the result). - let parts: Vec> = if let Some(repeat_matcher) = lookup_repeat_matcher(re) { - repeat_matcher.split(&str_data, limit) - } else if let Some(fre) = lookup_fancy_regex(re) { - crate::string::spec_fancy_regex_split(&fre, &str_data, limit) - } else { - // Standard engine: the JS `RegExp.prototype[Symbol.split]` algorithm - // (21.2.5.11). The `regex` crate's own `split` diverges from JS for - // zero-width matches (it emits leading/trailing/consecutive empty - // strings the spec's `e == p` skip suppresses) and never splices - // captured groups, so walk the string the spec's way instead. - crate::string::spec_regex_split(lazy::header_std_regex(re), &str_data, limit) - }; - - let arr = crate::array::js_array_alloc(parts.len() as u32); - let scope = crate::gc::RuntimeHandleScope::new(); - let arr_handle = scope.root_raw_mut_ptr(arr); - (*arr_handle.get_raw_mut_ptr::()).length = parts.len() as u32; - - for (i, part) in parts.iter().enumerate() { - let nanboxed = match part { - Some(text) => { - let str_ptr = js_string_from_str(text) as u64; - STRING_TAG | (str_ptr & POINTER_MASK) - } - None => TAG_UNDEFINED, - }; - let arr = arr_handle.get_raw_mut_ptr::(); - // GC_STORE_AUDIT(BARRIERED): regex split result slot uses the shared array slot-store helper. - crate::array::store_array_slot(arr, i, nanboxed); - } - arr_handle.get_raw_mut_ptr::() - } -} - -/// Search for a regex match in a string -/// string.search(regex) -> number (index of first match, -1 if none) -#[cfg(feature = "regex-engine")] -#[no_mangle] -pub extern "C" fn js_string_search_regex(s: *const StringHeader, re: *const RegExpHeader) -> i32 { - if !is_valid_ptr(s) || !is_valid_regex_ptr(re) { - return -1; - } - let str_data = string_as_str(s); - - unsafe { - if let Some(repeat_matcher) = lookup_repeat_matcher(re) { - return repeat_matcher - .regex - .find(str_data) - .map(|matched| byte_index_to_utf16_index(str_data, matched.start()) as i32) - .unwrap_or(-1); - } - - // Fancy-regex fallback (lookbehind/backreferences): the never-match - // placeholder in `regex_ptr` would always report -1 otherwise. - if let Some(fre) = lookup_fancy_regex(re) { - return match fre.find(str_data) { - Ok(Some(m)) => byte_index_to_utf16_index(str_data, m.start()) as i32, - _ => -1, - }; - } - - let regex = lazy::header_std_regex(re); - match regex.find(str_data) { - Some(m) => { - // `String.prototype.search` returns a JS string index — UTF-16 - // code units, matching `.index` / `lastIndex` / `str.length`. - byte_index_to_utf16_index(str_data, m.start()) as i32 - } - None => -1, - } - } -} - -/// Dynamic-receiver dispatch for `regex.test(str)` / `regex.exec(str)` when -/// codegen couldn't prove the receiver is a RegExp (e.g. hono's RegExpRouter -/// does `buildWildcardRegExp(k).test(path)`, where the receiver is the result -/// of a function call). Returns `Some(result)` only when `ptr` is a live regex -/// AND `method` is `test`/`exec`; `None` otherwise so the generic method -/// dispatch in `js_native_call_method` continues. The argument is coerced to a -/// string (`re.test(123)` tests against `"123"`). (#1731) -#[cfg(feature = "regex-engine")] pub(crate) fn dispatch_regex_receiver_method( ptr: *const u8, method: &str, diff --git a/crates/perry-runtime/src/regex/compile.rs b/crates/perry-runtime/src/regex/compile.rs index 6105ebc8da..f8472f50d6 100644 --- a/crates/perry-runtime/src/regex/compile.rs +++ b/crates/perry-runtime/src/regex/compile.rs @@ -150,13 +150,16 @@ pub extern "C" fn js_regexp_compile_value( // (see `REGEX_CACHE_MAX_ENTRIES`) can evict without invalidating this // receiver. Refresh `fancy_ptr` too — it must track the NEW pattern, not // the one the receiver was constructed with. - let arc = get_or_compile_regex(pattern_str, flags_str); + // `RegExp.prototype.compile` re-initialises an existing receiver — once per + // call from user code, not per object — so materialising the shared key + // here costs nothing measurable, and the same `Arc`s go into the source + // table below. + let pattern_key: std::sync::Arc = std::sync::Arc::from(pattern_str); + let flags_key: std::sync::Arc = std::sync::Arc::from(flags_str); + let arc = get_or_compile_regex(&pattern_key, &flags_key); let regex_ptr = Arc::into_raw(arc) as *mut Regex; let fancy_ptr: *const () = super::FANCY_CACHE.with(|fc| { - match fc - .borrow() - .get(&(pattern_str.to_string(), flags_str.to_string())) - { + match fc.borrow().get(&(pattern_key.clone(), flags_key.clone())) { Some(arc) => Arc::into_raw(arc.clone()) as *const (), None => std::ptr::null(), } @@ -164,7 +167,7 @@ pub extern "C" fn js_regexp_compile_value( let repeat_matcher_ptr: *const () = super::REPEAT_MATCHER_CACHE.with(|cache| { match cache .borrow() - .get(&(pattern_str.to_string(), flags_str.to_string())) + .get(&(pattern_key.clone(), flags_key.clone())) { Some(arc) => Arc::into_raw(arc.clone()) as *const (), None => std::ptr::null(), diff --git a/crates/perry-runtime/src/regex/exec.rs b/crates/perry-runtime/src/regex/exec.rs index dd09e6e44f..f0d760a39b 100644 --- a/crates/perry-runtime/src/regex/exec.rs +++ b/crates/perry-runtime/src/regex/exec.rs @@ -78,7 +78,9 @@ pub extern "C" fn js_regexp_exec( // `fancy_regex::Regex::captures_from_pos` and // `regress::Regex::find_from`. Their reported offsets are absolute, so // nothing downstream re-bases them. - let owned = if let Some(repeat_matcher) = lookup_repeat_matcher(re) { + let owned = if let Some(repeat_matcher) = + lookup_repeat_matcher_for(re, str_data, search_start_byte) + { repeat_matcher .regex .find_from(str_data, search_start_byte) @@ -212,7 +214,9 @@ pub(super) fn regexp_find_advancing( } else { 0 }; - let found = if let Some(repeat_matcher) = lookup_repeat_matcher(re) { + let found = if let Some(repeat_matcher) = + lookup_repeat_matcher_for(re, str_data, search_start_byte) + { repeat_matcher .regex .find_from(str_data, search_start_byte) diff --git a/crates/perry-runtime/src/regex/lazy.rs b/crates/perry-runtime/src/regex/lazy.rs index 236c7a80f4..a982a34875 100644 --- a/crates/perry-runtime/src/regex/lazy.rs +++ b/crates/perry-runtime/src/regex/lazy.rs @@ -233,7 +233,7 @@ fn build_and_install_programs(re: *const RegExpHeader) { let cache_hit = super::REGEX_CACHE.with(|cache| { cache .borrow() - .contains_key(&(pattern.to_string(), flags.to_string())) + .contains_key(&(pattern.clone(), flags.clone())) }); unsafe { let pattern_ptr = (*re).pattern_ptr; @@ -243,16 +243,13 @@ fn build_and_install_programs(re: *const RegExpHeader) { } } let std_arc = get_or_compile_regex(&pattern, &flags); - let fancy_arc: Option> = FANCY_CACHE.with(|fc| { - fc.borrow() - .get(&(pattern.to_string(), flags.to_string())) - .cloned() - }); + let fancy_arc: Option> = + FANCY_CACHE.with(|fc| fc.borrow().get(&(pattern.clone(), flags.clone())).cloned()); let repeat_arc: Option> = REPEAT_MATCHER_CACHE .with(|cache| { cache .borrow() - .get(&(pattern.to_string(), flags.to_string())) + .get(&(pattern.clone(), flags.clone())) .cloned() }); // ── Repair before publishing ────────────────────────────────────────── @@ -286,7 +283,7 @@ fn build_and_install_programs(re: *const RegExpHeader) { FANCY_CACHE.with(|fc| { let mut fc = fc.borrow_mut(); evict_regex_cache_if_full(&mut fc); - fc.insert((pattern.to_string(), flags.to_string()), arc.clone()); + fc.insert((pattern.clone(), flags.clone()), arc.clone()); }); fancy_arc = Some(arc); } @@ -301,7 +298,7 @@ fn build_and_install_programs(re: *const RegExpHeader) { REPEAT_MATCHER_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); evict_regex_cache_if_full(&mut cache); - cache.insert((pattern.to_string(), flags.to_string()), arc.clone()); + cache.insert((pattern.clone(), flags.clone()), arc.clone()); }); repeat_arc = Some(arc); } diff --git a/crates/perry-runtime/src/regex/match_all.rs b/crates/perry-runtime/src/regex/match_all.rs index 202e744609..998e2ad83f 100644 --- a/crates/perry-runtime/src/regex/match_all.rs +++ b/crates/perry-runtime/src/regex/match_all.rs @@ -97,7 +97,7 @@ unsafe fn materialize_match_all_results( let search_start = utf16_index_to_byte(str_data, start_char_index); let mut owned: Vec = Vec::new(); - if let Some(repeat_matcher) = super::lookup_repeat_matcher(re) { + if let Some(repeat_matcher) = super::lookup_repeat_matcher_for(re, str_data, search_start) { // `regress`'s own iterator is positional and already advances one // position past a zero-width match, which is the ECMAScript rule. for matched in repeat_matcher.regex.find_from(str_data, search_start) { diff --git a/crates/perry-runtime/src/regex/match_string.rs b/crates/perry-runtime/src/regex/match_string.rs index 6611923493..df4e49ad05 100644 --- a/crates/perry-runtime/src/regex/match_string.rs +++ b/crates/perry-runtime/src/regex/match_string.rs @@ -85,7 +85,7 @@ pub extern "C" fn js_string_match( let global = (*re).global; let has_indices = (*re).has_indices; - if let Some(repeat_matcher) = lookup_repeat_matcher(re) { + if let Some(repeat_matcher) = lookup_repeat_matcher_for(re, str_data, 0) { if global { let matches: Vec = repeat_matcher .regex diff --git a/crates/perry-runtime/src/regex/program_key.rs b/crates/perry-runtime/src/regex/program_key.rs new file mode 100644 index 0000000000..e753fe0f69 --- /dev/null +++ b/crates/perry-runtime/src/regex/program_key.rs @@ -0,0 +1,38 @@ +//! The never-match placeholder program and the compiled-program cache key. +//! +//! Split out of `regex.rs` to keep that file under the 2000-line size gate. + +use std::sync::Arc; + +/// The pattern of the never-match program `compile_and_cache_regex_checked` +/// installs in `REGEX_CACHE` for a pattern only `fancy-regex` accepts, so a +/// caller reaching for the standard program does not crash. +/// +/// Named because `lazy::build_and_install_programs` has to RECOGNISE it: a +/// header whose standard program is this placeholder is usable only through +/// its fancy fallback, so the fallback has to exist beside it. +#[cfg(feature = "regex-engine")] +pub(crate) const NEVER_MATCH_PATTERN: &str = r"[^\s\S]"; + +/// Key of the three compiled-program caches. +/// +/// `(String, String)` looks harmless and is not: `HashMap::get` needs a +/// `&(String, String)`, so **every probe materialised the key** — two heap +/// allocations and two copies of the pattern text, on a path that runs once +/// per RegExp OBJECT, and a JS regex literal evaluates to a fresh object every +/// time it is reached. A native-churn census of the claude-code binary +/// (2026-09-05) put `js_regexp_test` → `lookup_repeat_matcher` → +/// `build_and_install_programs` at **6,044 MB of 8,334 MB of estimated +/// allocation with zero live bytes** — 73 % of all remaining native churn — +/// split across the three probe sites: the `get_or_compile_regex` probe +/// (2,071 MB) and two `core::fmt::Formatter::pad` frames (1,989 MB and +/// 1,984 MB), which is what `.to_string()` on an `Arc` lowers to. +/// +/// Keying by `Arc` makes a probe two refcount increments and no +/// allocation: every caller that matters already holds those `Arc`s, because +/// `REGEX_SOURCE_TABLE` and `regex::site_cache` share one allocation of a +/// literal's text with every header built from it. Hashing still walks the +/// pattern bytes — the allocation is what the census measured, and what this +/// removes. +#[cfg(feature = "regex-engine")] +pub(crate) type ProgramKey = (Arc, Arc); diff --git a/crates/perry-runtime/src/regex/repeat_matcher.rs b/crates/perry-runtime/src/regex/repeat_matcher.rs index e2e19129c9..da008816de 100644 --- a/crates/perry-runtime/src/regex/repeat_matcher.rs +++ b/crates/perry-runtime/src/regex/repeat_matcher.rs @@ -241,7 +241,35 @@ fn quantifier_follows(bytes: &[u8], index: usize) -> bool { /// capture semantics. Besides quantified captures, this includes captures in a /// negative lookaround: after a successful negative assertion those captures /// are unmatched, so a later backreference must match the empty string. -fn quantified_capture_layout(pattern: &str) -> Option>> { +/// Is `regress` the PRIMARY engine for this process? +/// +/// `PERRY_REGEX_ENGINE=regress` routes every pattern through the ECMAScript +/// backtracker instead of only the ones whose RepeatMatcher capture semantics +/// are observable. It exists to MEASURE the tier-0 engine architecture end to +/// end — compile cost, bytes of program, and the match-time cost of giving up +/// the linear engine — in a real binary on the real rig rather than only in a +/// corpus harness. It is NOT a supported configuration: the backtracker has no +/// step budget, so a pathological pattern can run unbounded. +pub(super) fn regress_first() -> bool { + use std::sync::atomic::{AtomicU8, Ordering}; + static STATE: AtomicU8 = AtomicU8::new(0); + match STATE.load(Ordering::Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + let on = std::env::var("PERRY_REGEX_ENGINE") + .map(|v| v.eq_ignore_ascii_case("regress")) + .unwrap_or(false); + STATE.store(if on { 2 } else { 1 }, Ordering::Relaxed); + on +} + +/// The capture-name layout of `pattern`, and whether ECMA-262's RepeatMatcher +/// capture-reset semantics are OBSERVABLE for it (a capture group directly +/// under a quantifier, or a capture inside a negative lookaround — the two +/// shapes where the linear engine's answer differs from the spec's). +fn capture_layout(pattern: &str) -> (Vec>, bool) { let bytes = pattern.as_bytes(); let mut captures = Vec::new(); let mut groups = Vec::new(); @@ -288,11 +316,14 @@ fn quantified_capture_layout(pattern: &str) -> Option>> { _ => index += 1, } } - needs_repeat_matcher.then_some(captures) + (captures, needs_repeat_matcher) } pub(super) fn compile(pattern: &str, flags: &str) -> Option { - let capture_names = quantified_capture_layout(pattern)?; + let (capture_names, needs_repeat_matcher) = capture_layout(pattern); + if !needs_repeat_matcher && !regress_first() { + return None; + } let regex = regress::Regex::with_flags(pattern, flags).ok()?; Some(RepeatMatcherRegex { regex, @@ -463,20 +494,20 @@ mod tests { #[test] fn detects_only_quantified_groups_with_captures() { - assert!(quantified_capture_layout(r"(a?b??)*").is_some()); - assert!(quantified_capture_layout(r"(?:(?=(abc))){0,1}a").is_some()); - assert!(quantified_capture_layout(r"(?!(a)b)\1").is_some()); - assert!(quantified_capture_layout(r"(?a)(b))*"), - Some(vec![Some("first".to_string()), None]) + capture_layout(r"(?:(?a)(b))*"), + (vec![Some("first".to_string()), None], true) ); } } diff --git a/crates/perry-runtime/src/regex/replace_expand.rs b/crates/perry-runtime/src/regex/replace_expand.rs index 4b87e5a0fd..70b1931f61 100644 --- a/crates/perry-runtime/src/regex/replace_expand.rs +++ b/crates/perry-runtime/src/regex/replace_expand.rs @@ -3,6 +3,7 @@ //! `expand_js_replacement` (ECMAScript `$`-pattern expansion) and //! `replace_regex_fn_fancy` (the fancy-regex callback-replace fallback). +use super::replace_expand_fancy::replace_regex_str_fancy; use super::replace_fn::{copy_replace_source, finish_replace_bytes}; use super::*; @@ -477,7 +478,7 @@ pub extern "C" fn js_string_replace_regex_named( } unsafe { - if let Some(repeat_matcher) = lookup_repeat_matcher(re) { + if let Some(repeat_matcher) = lookup_repeat_matcher_for(re, str_data, 0) { let result = repeat_matcher.replace(str_data, repl_str, (*re).global); return finish_replace_bytes(result.as_bytes()); } diff --git a/crates/perry-runtime/src/regex/replace_expand_fancy.rs b/crates/perry-runtime/src/regex/replace_expand_fancy.rs new file mode 100644 index 0000000000..22e78cee6a --- /dev/null +++ b/crates/perry-runtime/src/regex/replace_expand_fancy.rs @@ -0,0 +1,390 @@ +//! `String.prototype.replace` substitution expansion for the fancy-regex path. +//! +//! Split out of `regex.rs` to keep that file under the 2000-line size gate. + +use super::*; + +pub(super) fn expand_js_replacement_fancy( + repl: &str, + caps: &fancy_regex::Captures, + subject: &str, + has_named_groups: bool, +) -> String { + let m0 = match caps.get(0) { + Some(m) => m, + None => return String::new(), + }; + let (mstart, mend) = (m0.start(), m0.end()); + let ngroups = caps.len(); + let b = repl.as_bytes(); + let mut out = String::with_capacity(repl.len() + 16); + let mut i = 0; + while i < b.len() { + if b[i] != b'$' { + let start = i; + while i < b.len() && b[i] != b'$' { + i += 1; + } + out.push_str(&repl[start..i]); + continue; + } + if i + 1 >= b.len() { + out.push('$'); + i += 1; + continue; + } + match b[i + 1] { + b'$' => { + out.push('$'); + i += 2; + } + b'&' => { + out.push_str(&subject[mstart..mend]); + i += 2; + } + b'`' => { + out.push_str(&subject[..mstart]); + i += 2; + } + b'\'' => { + out.push_str(&subject[mend..]); + i += 2; + } + b'0'..=b'9' => { + let d1 = (b[i + 1] - b'0') as usize; + let (group, consumed) = if i + 2 < b.len() && b[i + 2].is_ascii_digit() { + let two = d1 * 10 + (b[i + 2] - b'0') as usize; + if two >= 1 && two < ngroups { + (Some(two), 2) + } else if d1 >= 1 && d1 < ngroups { + (Some(d1), 1) + } else { + (None, 0) + } + } else if d1 >= 1 && d1 < ngroups { + (Some(d1), 1) + } else { + (None, 0) + }; + match group { + Some(g) => { + if let Some(m) = caps.get(g) { + out.push_str(m.as_str()); + } + i += 1 + consumed; + } + None => { + out.push('$'); + i += 1; + } + } + } + b'<' if has_named_groups => { + if let Some(rel) = repl[i + 2..].find('>') { + let name = &repl[i + 2..i + 2 + rel]; + if let Some(m) = caps.name(name) { + out.push_str(m.as_str()); + } + i += 2 + rel + 1; + } else { + out.push('$'); + i += 1; + } + } + _ => { + out.push('$'); + i += 1; + } + } + } + out +} + +/// Fancy-regex fallback for the string-replacement (non-callback) forms of +/// `String.prototype.replace`/`replaceAll`. Drives a manual non-overlapping +/// match loop with `fancy_regex` and expands the replacement string via +/// [`expand_js_replacement_fancy`]. Used when the pattern needs +/// lookbehind/backreferences the `regex` crate can't compile. +#[cfg(feature = "regex-engine")] +pub(super) unsafe fn replace_regex_str_fancy( + str_data: &str, + fre: &fancy_regex::Regex, + global: bool, + repl_str: &str, +) -> *mut StringHeader { + let has_named_groups = fre.capture_names().any(|n| n.is_some()); + // #9430: the ECMAScript scan for the global form. fancy-regex's own + // iterator drops a zero-width match that lands where the previous match + // ended, so `"a".replace(/(?<=x)?a*/g, …)`-shaped patterns lost their + // trailing (and every interior) empty replacement. + let captures_list: Vec = if global { + global_scan::fancy_captures(fre, str_data, 0) + } else { + match fre.captures(str_data) { + Ok(Some(caps)) => vec![caps], + Ok(None) | Err(_) => Vec::new(), + } + }; + let mut result = String::new(); + let mut last_end = 0usize; + for caps in &captures_list { + let full_match = caps.get(0).unwrap(); + result.push_str(&str_data[last_end..full_match.start()]); + result.push_str(&expand_js_replacement_fancy( + repl_str, + caps, + str_data, + has_named_groups, + )); + last_end = full_match.end(); + } + result.push_str(&str_data[last_end..]); + finish_replace_bytes(result.as_bytes()) +} + +/// string.replace(regex, replacement) -> string +#[cfg(feature = "regex-engine")] +#[no_mangle] +pub extern "C" fn js_string_replace_regex( + s: *const StringHeader, + re: *const RegExpHeader, + replacement: *const StringHeader, +) -> *mut StringHeader { + if !is_valid_ptr(s) { + return js_string_from_str(""); + } + + if !is_valid_regex_ptr(re) { + // If regex is null, return original string + return copy_replace_source(s); + } + if crate::hot_diag::regex_on() { + diag_note_op(re, crate::hot_diag::RegexOp::Replace); + } + + unsafe { + // The Rust string engines require scalar-value UTF-8, while Perry + // stores lone JavaScript surrogates as WTF-8. Match those subjects as + // UTF-16 code units with the ECMAScript engine and rebuild the result + // through the WTF-8-aware string builder. + if (*s).flags & crate::string::STRING_FLAG_HAS_LONE_SURROGATES != 0 { + let replacement_bytes = if is_valid_ptr(replacement) { + string_as_bytes(replacement) + } else { + b"undefined" + }; + if let Some(result) = repeat_matcher::replace_wtf8_subject( + re, + string_as_bytes(s), + replacement_bytes, + (*re).global, + ) { + return finish_replace_bytes(&result); + } + } + + let str_data = string_as_str(s); + let repl_str = if is_valid_ptr(replacement) { + string_as_str(replacement) + } else { + "undefined" + }; + + if let Some(repeat_matcher) = lookup_repeat_matcher_for(re, str_data, 0) { + let result = repeat_matcher.replace(str_data, repl_str, (*re).global); + return finish_replace_bytes(result.as_bytes()); + } + + // Pattern the `regex` crate couldn't compile (lookbehind/backreferences) + // → drive the replacement through fancy-regex. Otherwise the never-match + // placeholder in `regex_ptr` would leave the input unchanged. + if let Some(fre) = lookup_fancy_regex(re) { + return replace_regex_str_fancy(str_data, &fre, (*re).global, repl_str); + } + + let regex = lazy::header_std_regex(re); + let global = (*re).global; + let has_named_groups = regex.capture_names().any(|n| n.is_some()); + + // Route through a JS-aware expander (closure form) so `$&` / `` $` `` / + // `$'` — which the regex crate's native `$` syntax doesn't support — + // are substituted per match. `$$`, `$n`, and `$` are handled too. + // #9430: `Regex::replace_all` runs the crate's own match iterator, + // whose empty-match rule is not ECMAScript's. Drive the ECMAScript + // scan and splice the replacements here instead; the non-global form + // is the same loop over a one-element list. + let captures_list: Vec = if global { + global_scan::std_captures(regex, str_data, 0) + } else { + regex.captures(str_data).into_iter().collect() + }; + if crate::hot_diag::regex_on() { + let n = captures_list.len() as u64; + crate::hot_diag::regex_with(|d| d.replace_matches += n); + } + let mut result = String::with_capacity(str_data.len()); + let mut last_end = 0usize; + for caps in &captures_list { + let full = caps.get(0).expect("capture zero is the full match"); + result.push_str(&str_data[last_end..full.start()]); + result.push_str(&expand_js_replacement( + repl_str, + caps, + str_data, + has_named_groups, + )); + last_end = full.end(); + } + result.push_str(&str_data[last_end..]); + + finish_replace_bytes(result.as_bytes()) + } +} + +/// string.replaceAll(regex, replacement) -> string +#[cfg(feature = "regex-engine")] +#[no_mangle] +pub extern "C" fn js_string_replace_all_regex( + s: *const StringHeader, + re: *const RegExpHeader, + replacement: *const StringHeader, +) -> *mut StringHeader { + if !is_valid_ptr(s) { + return js_string_from_str(""); + } + + if !is_valid_regex_ptr(re) { + return copy_replace_source(s); + } + + ensure_replace_all_regex_global(re); + js_string_replace_regex(s, re, replacement) +} + +/// Split a string by a regex delimiter +/// string.split(regex) -> string[] (array of NaN-boxed string pointers) +#[cfg(feature = "regex-engine")] +#[no_mangle] +pub extern "C" fn js_string_split_regex( + s: *const StringHeader, + re: *const RegExpHeader, +) -> *mut ArrayHeader { + js_string_split_regex_n(s, re, -1) +} + +/// string.split(regex, limit) — limit<0 means no limit, limit==0 means empty +/// (issue #567). +#[cfg(feature = "regex-engine")] +#[no_mangle] +pub extern "C" fn js_string_split_regex_n( + s: *const StringHeader, + re: *const RegExpHeader, + limit: i32, +) -> *mut ArrayHeader { + const STRING_TAG: u64 = 0x7FFF_0000_0000_0000; + const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; + + if !is_valid_ptr(s) { + return crate::array::js_array_alloc(0); + } + if limit == 0 { + return crate::array::js_array_alloc(0); + } + let str_data = string_as_str(s).to_owned(); + + if !is_valid_regex_ptr(re) { + // No regex: return array with the whole string as a single element + let arr = crate::array::js_array_alloc(1); + let scope = crate::gc::RuntimeHandleScope::new(); + let arr_handle = scope.root_raw_mut_ptr(arr); + // Allocating string + array re-read as one combinator (#7341). + let (str_ptr, arr) = + arr_handle.across_mut::(|| js_string_from_str(&str_data) as u64); + unsafe { + (*arr).length = 1; + let nanboxed = STRING_TAG | (str_ptr & POINTER_MASK); + // GC_STORE_AUDIT(BARRIERED): regex split fallback slot uses the shared array slot-store helper. + crate::array::store_array_slot(arr, 0, nanboxed); + } + return arr_handle.with_mut_ptr::(|a| a); + } + + const TAG_UNDEFINED: u64 = 0x7FFC_0000_0000_0001; + unsafe { + // Each element is either a substring (`Some`) or `undefined` (`None`, + // for an unmatched capture group spliced into the result). + let parts: Vec> = + if let Some(repeat_matcher) = lookup_repeat_matcher_for(re, &str_data, 0) { + repeat_matcher.split(&str_data, limit) + } else if let Some(fre) = lookup_fancy_regex(re) { + crate::string::spec_fancy_regex_split(&fre, &str_data, limit) + } else { + // Standard engine: the JS `RegExp.prototype[Symbol.split]` algorithm + // (21.2.5.11). The `regex` crate's own `split` diverges from JS for + // zero-width matches (it emits leading/trailing/consecutive empty + // strings the spec's `e == p` skip suppresses) and never splices + // captured groups, so walk the string the spec's way instead. + crate::string::spec_regex_split(lazy::header_std_regex(re), &str_data, limit) + }; + + let arr = crate::array::js_array_alloc(parts.len() as u32); + let scope = crate::gc::RuntimeHandleScope::new(); + let arr_handle = scope.root_raw_mut_ptr(arr); + arr_handle.with_mut_ptr::(|a| (*a).length = parts.len() as u32); + + for (i, part) in parts.iter().enumerate() { + let nanboxed = match part { + Some(text) => { + let str_ptr = js_string_from_str(text) as u64; + STRING_TAG | (str_ptr & POINTER_MASK) + } + None => TAG_UNDEFINED, + }; + // Re-read per iteration: `js_string_from_str` above allocates. + let arr = arr_handle.with_mut_ptr::(|a| a); + // GC_STORE_AUDIT(BARRIERED): regex split result slot uses the shared array slot-store helper. + crate::array::store_array_slot(arr, i, nanboxed); + } + arr_handle.with_mut_ptr::(|a| a) + } +} + +/// Search for a regex match in a string +/// string.search(regex) -> number (index of first match, -1 if none) +#[cfg(feature = "regex-engine")] +#[no_mangle] +pub extern "C" fn js_string_search_regex(s: *const StringHeader, re: *const RegExpHeader) -> i32 { + if !is_valid_ptr(s) || !is_valid_regex_ptr(re) { + return -1; + } + let str_data = string_as_str(s); + + unsafe { + if let Some(repeat_matcher) = lookup_repeat_matcher_for(re, str_data, 0) { + return repeat_matcher + .regex + .find(str_data) + .map(|matched| byte_index_to_utf16_index(str_data, matched.start()) as i32) + .unwrap_or(-1); + } + + // Fancy-regex fallback (lookbehind/backreferences): the never-match + // placeholder in `regex_ptr` would always report -1 otherwise. + if let Some(fre) = lookup_fancy_regex(re) { + return match fre.find(str_data) { + Ok(Some(m)) => byte_index_to_utf16_index(str_data, m.start()) as i32, + _ => -1, + }; + } + + let regex = lazy::header_std_regex(re); + match regex.find(str_data) { + Some(m) => { + // `String.prototype.search` returns a JS string index — UTF-16 + // code units, matching `.index` / `lastIndex` / `str.length`. + byte_index_to_utf16_index(str_data, m.start()) as i32 + } + None => -1, + } + } +} diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index d82dea78e9..2c571153c4 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -905,7 +905,10 @@ fn regex_cache_capped_and_prior_headers_survive_eviction() { // Flood the cache with distinct patterns — far past the cap. for i in 0..(REGEX_CACHE_MAX_ENTRIES * 2 + 10) { - let _ = get_or_compile_regex(&format!("cachefill{i}[a-z]+"), ""); + let _ = get_or_compile_regex( + &Arc::from(format!("cachefill{i}[a-z]+").as_str()), + &Arc::from(""), + ); } let std_len = REGEX_CACHE.with(|c| c.borrow().len()); assert!( @@ -915,7 +918,10 @@ fn regex_cache_capped_and_prior_headers_survive_eviction() { // Flood the fancy cache as well (each pattern rejected by the std engine). for i in 0..(REGEX_CACHE_MAX_ENTRIES + 10) { - let _ = get_or_compile_regex(&format!("(?<=fill{i})x"), ""); + let _ = get_or_compile_regex( + &Arc::from(format!("(?<=fill{i})x").as_str()), + &Arc::from(""), + ); } let fancy_len = FANCY_CACHE.with(|c| c.borrow().len()); assert!( @@ -925,7 +931,7 @@ fn regex_cache_capped_and_prior_headers_survive_eviction() { // Quantified captures populate the ECMAScript RepeatMatcher cache. for i in 0..(REGEX_CACHE_MAX_ENTRIES + 10) { - let _ = get_or_compile_regex(&format!("(repeat{i})*"), ""); + let _ = get_or_compile_regex(&Arc::from(format!("(repeat{i})*").as_str()), &Arc::from("")); } let repeat_len = REPEAT_MATCHER_CACHE.with(|c| c.borrow().len()); assert!( @@ -1807,7 +1813,7 @@ fn a_single_program_cache_clear_cannot_disarm_a_lookbehind_literal() { assert!( REGEX_CACHE.with(|c| c .borrow() - .contains_key(&(source.to_string(), String::new()))), + .contains_key(&(std::sync::Arc::from(source), std::sync::Arc::from("")))), "the placeholder must survive, or this test exercises nothing" ); // A fresh literal site, so the construction cache cannot answer from the @@ -1830,3 +1836,51 @@ fn a_single_program_cache_clear_cannot_disarm_a_lookbehind_literal() { "the literal must still match after an unrelated cache reached capacity" ); } + +/// The backtracking cliff: a capture group under a quantifier takes a pattern +/// off the linear engine, and the ECMAScript backtracker has no step budget. +/// `/^(a+)+$/.test("a"*28 + "!")` measured 16.5 s against 4.8 s for node and +/// 0 ms for the identical-language `/^(?:a+)+$/`. +/// +/// The linear program proves the answer in O(n) — the two engines accept the +/// same language and disagree only about capture ASSIGNMENT — so the +/// backtracker must not be entered for a subject the linear engine has already +/// ruled out. This test would take minutes without that gate. +#[test] +fn quantified_capture_pattern_does_not_backtrack_on_a_non_matching_subject() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let pattern = scope.root_string_ptr(make_string("^(a+)+$")); + let flags = scope.root_string_ptr(make_string("")); + let re = pattern.with_mut_ptr::(|pattern| { + flags.with_mut_ptr::(|flags| js_regexp_new(pattern, flags)) + }); + // The pattern really is on the backtracker — that is the premise. + assert!( + lookup_repeat_matcher(re).is_some(), + "a capture under a quantifier must route to the ECMAScript matcher" + ); + + let hay = format!("{}!", "a".repeat(40)); + let subject = scope.root_string_ptr(make_string(&hay)); + let started = std::time::Instant::now(); + assert_eq!( + subject.with_const_ptr::(|s| js_regexp_test(re, s)), + 0, + "no match: the subject ends in '!'" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "a non-matching subject must not be handed to the backtracker \ + (took {:?} for 40 characters)", + started.elapsed() + ); + + // A subject that DOES match still goes through the backtracker and still + // reports the spec's captures. + let good = scope.root_string_ptr(make_string("aaaa")); + assert_eq!( + good.with_const_ptr::(|s| js_regexp_test(re, s)), + 1 + ); +} diff --git a/crates/perry-runtime/src/string/char_ops.rs b/crates/perry-runtime/src/string/char_ops.rs index 9bdf5cbba0..6ef47c734e 100644 --- a/crates/perry-runtime/src/string/char_ops.rs +++ b/crates/perry-runtime/src/string/char_ops.rs @@ -253,12 +253,16 @@ pub extern "C" fn js_string_char_at(s: *const StringHeader, index: i32) -> *mut return js_string_from_bytes(std::ptr::null(), 0); } - // ASCII fast path: skip utf16_len scan + // ASCII fast path: skip utf16_len scan. The result is one of exactly 128 + // possible strings, so it comes from the canonical per-thread table + // (`ascii_char_string`) instead of being minted: `s[i]` / `charAt` / + // `[...s]` / every runtime character walk stops allocating. The table's + // entries are `refcount = 0` (shared, never mutated in place), which is + // what makes returning the same pointer to every caller sound. if is_ascii_string(s) { unsafe { let data = string_data(s); - let char_ptr = data.add(index as usize); - return js_string_from_ascii_bytes(char_ptr, 1); + return crate::string::ascii_char_string(*data.add(index as usize)); } } @@ -369,8 +373,9 @@ fn encode_3byte_wtf8(unit: u16) -> [u8; 3] { /// for the old `char::from_u32(..).unwrap_or('\u{FFFD}')` lossy path. pub(crate) fn string_from_code_unit(unit: u16) -> *mut StringHeader { if unit < 0x80 { - let byte = unit as u8; - return js_string_from_bytes(&byte as *const u8, 1); + // Canonical table (see `ascii_char_string`): a one-ASCII-character + // string has 128 possible contents and is never mutated in place. + return crate::string::ascii_char_string(unit as u8); } if (0xD800..=0xDFFF).contains(&unit) { let buf = encode_3byte_wtf8(unit); diff --git a/crates/perry-runtime/src/string/format.rs b/crates/perry-runtime/src/string/format.rs index 2e46dc83e4..0d106aaab9 100644 --- a/crates/perry-runtime/src/string/format.rs +++ b/crates/perry-runtime/src/string/format.rs @@ -17,6 +17,52 @@ crate::perry_thread_local! { const { std::cell::UnsafeCell::new([std::ptr::null_mut(); SMALL_INT_CACHE_SIZE]) }; } +/// Cached single-ASCII-character string table (`"\0"`..`"\x7f"`), the exact +/// analogue of [`SMALL_INT_CACHE`] one dimension over: every `s[i]`, +/// `s.charAt(i)`, `[...s]` and every runtime consumer of +/// [`js_string_char_at`](super::js_string_char_at) used to MINT a fresh +/// 32-byte heap string per character read. On the compiled claude-code TUI — +/// which measures, wraps and ANSI-scans every rendered line — that is one of +/// the largest single contributors to allocation volume, and the bytes are +/// pure garbage: a one-character ASCII string has exactly 128 possible +/// contents. +/// +/// Same residency contract as `SMALL_INT_CACHE`, and for the same reasons: +/// per-thread (arena pointers are not shareable), longlived-arena (so the +/// entry never anchors a nursery block), `refcount = 0` (shared — never +/// mutated in place, which is what makes handing the SAME pointer to every +/// caller sound), pinned out of the young generation, and scanned by +/// [`scan_small_int_cache_roots_mut`] so the collector rewrites the slot if +/// the longlived object is ever relocated. +const ASCII_CHAR_CACHE_SIZE: usize = 128; +crate::perry_thread_local! { + static ASCII_CHAR_CACHE: std::cell::UnsafeCell<[*mut StringHeader; ASCII_CHAR_CACHE_SIZE]> = + const { std::cell::UnsafeCell::new([std::ptr::null_mut(); ASCII_CHAR_CACHE_SIZE]) }; +} + +/// The canonical one-character string for an ASCII byte. Allocates at most +/// once per byte value per thread; every later call is a load. +pub(crate) fn ascii_char_string(byte: u8) -> *mut StringHeader { + debug_assert!(byte < 0x80); + let idx = (byte & 0x7f) as usize; + let cached = ASCII_CHAR_CACHE.with(|c| unsafe { (*c.get())[idx] }); + if !cached.is_null() { + return cached; + } + let ptr = js_string_from_bytes_longlived(&byte as *const u8, 1); + unsafe { + (*ptr).refcount = 0; + let gc_header = + (ptr as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *mut crate::gc::GcHeader; + crate::gc::pin_object_non_young(gc_header); + } + ASCII_CHAR_CACHE.with(|c| unsafe { + // GC_STORE_AUDIT(ROOT): ASCII_CHAR_CACHE is scanned by scan_small_int_cache_roots_mut. + crate::gc::runtime_store_root_raw_mut_ptr_slot(&raw mut (*c.get())[idx], ptr); + }); + ptr +} + /// Normalize a `Number.prototype` format-method receiver to its underlying /// `f64`. Codegen lowers `x.toFixed(n)` / `.toExponential(n)` / `.toPrecision(n)` /// to a direct runtime call that passes the receiver's bits as the first `f64` @@ -162,6 +208,19 @@ pub fn scan_small_int_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisito } } }); + // The single-character table rides the same scanner rather than + // registering a 96th root scanner: both are per-thread arrays of + // canonical `StringHeader*` with identical residency rules, and the + // per-collection cost of every additional registered scanner is the + // thing the collector is trying to shed. + ASCII_CHAR_CACHE.with(|c| unsafe { + for slot in (*c.get()).iter_mut() { + let mut addr = *slot as usize; + if visitor.visit_tagged_usize_slot(&mut addr, crate::value::STRING_TAG) { + *slot = addr as *mut StringHeader; + } + } + }); } fn is_undefined_arg(value: f64) -> bool { diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index f673d45d22..bb87bd598c 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -165,8 +165,34 @@ pub use concat::{ scan_concat_memo_roots, scan_concat_memo_roots_mut, }; pub use concat_site::{js_string_concat_site_value, CONCAT_SITE_SLOTS}; +pub(crate) use format::ascii_char_string; pub(crate) use format::fix_exponent_format; pub(crate) use format::js_format_f64; + +/// The canonical `StringHeader` for a runtime-internal constant property name. +/// +/// Perry's runtime resolves fixed names — `"constructor"`, `"prototype"`, +/// `"toString"`, the `globalThis` builtin a primitive method call dispatches +/// through — by MINTING a fresh heap string for the literal on every lookup +/// and throwing it away one call later. On the compiled claude-code TUI that +/// is measured in hundreds of megabytes per reply +/// (`js_get_global_this_builtin_value` alone: 133 MB of the 990 MB a +/// 3300-character reply allocates), all of it identical bytes. +/// +/// This routes those literals through the intern table that already exists for +/// exactly this purpose (`js_string_materialize_to_heap` uses it for computed +/// property names): content-keyed, per-thread, allocated once, address-stable, +/// `refcount = 0` and `GC_FLAG_INTERNED` so nothing mutates it in place — and +/// already covered by the intern-table root scanner, so it adds no new root +/// surface. A key that is interned also makes the property-read and +/// property-write fast paths eligible, which the freshly minted copy never was. +/// +/// Use it only for names the runtime itself spells as a literal. A key built +/// from user data belongs on the ordinary allocation path. +#[inline] +pub(crate) fn canonical_key(name: &[u8]) -> *mut StringHeader { + intern::intern_dispatch_bytes(0, name.as_ptr(), name.len(), 0, false) as *mut StringHeader +} pub use format::{ js_number_to_exponential, js_number_to_fixed, js_number_to_precision, js_number_to_string, scan_small_int_cache_roots, scan_small_int_cache_roots_mut, diff --git a/crates/perry-runtime/src/string/tests.rs b/crates/perry-runtime/src/string/tests.rs index c001ba15d9..2888700868 100644 --- a/crates/perry-runtime/src/string/tests.rs +++ b/crates/perry-runtime/src/string/tests.rs @@ -1297,6 +1297,68 @@ mod split_empty_delimiter_code_units { } } +/// The canonical one-ASCII-character string table (`string::format`). +#[cfg(test)] +mod canonical_char_cache { + use super::*; + + /// A one-ASCII-character string has exactly 128 possible contents, so + /// `js_string_char_at` (and everything that funnels through it: `s[i]`, + /// `charAt`, `[...s]`, the String-wrapper index installer) hands back the + /// canonical per-thread header instead of minting one per read. + /// + /// The identity assertion is the whole point — it is what makes the + /// allocation disappear — and it fails the moment the canonical table is + /// bypassed. The `refcount == 0` assertion is the safety half: a shared + /// header must never be eligible for the in-place append optimisation. + #[test] + fn ascii_char_at_returns_one_canonical_shared_header_per_byte() { + let scope = crate::gc::RuntimeHandleScope::new(); + let s = scope.root_string_ptr(js_string_from_bytes(b"abca".as_ptr(), 4)); + let (a0, b1, a3) = s.with_const_ptr::(|s| { + ( + js_string_char_at(s, 0), + js_string_char_at(s, 1), + js_string_char_at(s, 3), + ) + }); + assert_eq!(a0, a3, "the same character must reuse the canonical header"); + assert_ne!(a0, b1, "different characters are different headers"); + unsafe { + assert_eq!((*a0).byte_len, 1); + assert_eq!((*a0).utf16_len, 1); + let data = (a0 as *const u8).add(std::mem::size_of::()); + assert_eq!(*data, b'a'); + assert_eq!( + (*a0).refcount, + 0, + "a shared header must be ineligible for the in-place append path" + ); + } + // A second string with the same character resolves to the same header: + // the table is keyed by content, not by source string. + let other = scope.root_string_ptr(js_string_from_bytes(b"za".as_ptr(), 2)); + let a_again = other.with_const_ptr::(|o| js_string_char_at(o, 1)); + assert_eq!(a0, a_again); + } + + /// Non-ASCII keeps the minting path (the canonical table is ASCII-only), + /// and the value is still correct — the fast path must not answer for + /// characters it does not represent. + #[test] + fn non_ascii_char_at_is_unaffected_by_the_canonical_table() { + let scope = crate::gc::RuntimeHandleScope::new(); + let s = scope.root_string_ptr(js_string_from_bytes("aé".as_ptr(), 3)); + let (c0, c1) = s.with_const_ptr::(|s| { + (js_string_char_at(s, 0), js_string_char_at(s, 1)) + }); + unsafe { + assert_eq!((*c0).byte_len, 1); + assert_eq!((*c1).byte_len, 2, "é is two UTF-8 bytes"); + } + } +} + /// `header_str_checked` answers exactly like `from_utf8(..).ok()` — a pure /// ASCII key without the scan, a non-ASCII scalar key by validation, and a /// WTF-8 payload (lone surrogate) as `None`. diff --git a/crates/perry-runtime/src/value/to_string.rs b/crates/perry-runtime/src/value/to_string.rs index b6a2b7b8d9..6129eaeee6 100644 --- a/crates/perry-runtime/src/value/to_string.rs +++ b/crates/perry-runtime/src/value/to_string.rs @@ -623,8 +623,7 @@ unsafe fn array_prototype_to_string_override(value: f64) -> ArrayToStringOutcome // collector and re-read its address after every allocation. let scope = crate::gc::RuntimeHandleScope::new(); let value_handle = scope.root_nanbox_f64(value); - let key_handle = - scope.root_string_ptr(crate::string::js_string_from_bytes(b"toString".as_ptr(), 8)); + let key_handle = scope.root_string_ptr(crate::string::canonical_key(b"toString")); let proto = crate::object::builtin_prototype_value("Array"); let proto_handle = scope.root_nanbox_f64(proto); let proto_bits = proto_handle.get_nanbox_f64().to_bits(); @@ -690,8 +689,7 @@ pub(crate) fn call_array_prototype_to_string_method( unsafe { let scope = crate::gc::RuntimeHandleScope::new(); let receiver_handle = scope.root_nanbox_f64(value); - let key_handle = - scope.root_string_ptr(crate::string::js_string_from_bytes(b"toString".as_ptr(), 8)); + let key_handle = scope.root_string_ptr(crate::string::canonical_key(b"toString")); let prototype_handle = scope.root_nanbox_f64(crate::object::builtin_prototype_value("Array")); let prototype_bits = prototype_handle.get_nanbox_f64().to_bits(); @@ -790,7 +788,7 @@ unsafe fn call_method_for_primitive( if obj_ptr.is_null() || (obj_ptr as usize) < 0x10000 { return MethodOutcome::Absent; } - let key = crate::string::js_string_from_bytes(method_name.as_ptr(), method_name.len() as u32); + let key = crate::string::canonical_key(method_name); let key_handle = scope.root_string_ptr(key); // Presence is independent from the value returned by Get. In particular, // an inherited accessor may exist yet return undefined/null; that is a @@ -870,7 +868,7 @@ unsafe fn call_function_method( return FunctionMethodOutcome::Absent; } - let key = crate::string::js_string_from_bytes(method_name.as_ptr(), method_name.len() as u32); + let key = crate::string::canonical_key(method_name); let key_handle = scope.root_string_ptr(key); let key_ptr = key_handle.get_raw_const_ptr::(); let method = function_method_value(closure_ptr, key_ptr, method_name); diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index ae988212c7..3fc67c1a83 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -45,6 +45,9 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_RS4GC", // `-Os` vs `-O3` for every native module. "PERRY_LL_SIZE_OPT", + // Explicit application-module LLVM optimization level. This overrides the + // normal `PERRY_LL_SIZE_OPT` selection and changes every emitted object. + "PERRY_LL_OPT_LEVEL", // The post-RS4GC per-function instruction budget (#8583/#8679): a function // one setting re-lowers must not be served from a build another kept on // statepoints. @@ -53,6 +56,10 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ // `disable-tail-calls` before the optimizer. It changes the generated // code of the functions it trips on, so it is a cache input. "PERRY_LL_TRE_MAX_ALLOCA_WALK", + // A unit over this post-optimization per-function ceiling uses LLVM's O0 + // machine pipeline for bounded ISel/regalloc. That changes object bytes, + // so both the build and object caches must distinguish its settings. + "PERRY_LL_FAST_EMIT_MAX_INSTRS", // #9071: gates resolving a loop-called immutable callee binding once at // body entry instead of per call — the two settings emit different call // sequences, so a cached object from one must not serve the other. diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index 79fd77520c..6274c7a400 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -1056,6 +1056,10 @@ fn compute_object_cache_key_with_env( "env_ll_size_opt", env_var("PERRY_LL_SIZE_OPT").as_deref().unwrap_or(""), ); + h.field( + "env_ll_opt_level", + env_var("PERRY_LL_OPT_LEVEL").as_deref().unwrap_or(""), + ); // #8583/#8679: the post-RS4GC instruction budget decides whether functions // are re-lowered onto shadow frames; two settings must never share an object. h.field( @@ -1072,6 +1076,15 @@ fn compute_object_cache_key_with_env( .as_deref() .unwrap_or(""), ); + // Oversized post-optimization functions can use LLVM's O0 machine + // pipeline for bounded ISel/regalloc; the resulting object differs from + // normal optimized machine emission. + h.field( + "env_ll_fast_emit_max_instrs", + env_var("PERRY_LL_FAST_EMIT_MAX_INSTRS") + .as_deref() + .unwrap_or(""), + ); // #8583: root-spill threshold changes which functions carry statepoints. h.field( "env_root_spill_relocations", diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index aa2d196db1..ac61b79215 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -731,8 +731,10 @@ fn key_changes_with_codegen_env_vars() { "PERRY_SHADOW_STACK", "PERRY_RS4GC", "PERRY_LL_SIZE_OPT", + "PERRY_LL_OPT_LEVEL", "PERRY_LL_RS4GC_MAX_INSTRS", "PERRY_LL_TRE_MAX_ALLOCA_WALK", + "PERRY_LL_FAST_EMIT_MAX_INSTRS", "PERRY_ROOT_SPILL_RELOCATIONS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_DISABLE_BUFFER_FAST_PATH", diff --git a/crates/perry/src/commands/compile/run_pipeline.rs b/crates/perry/src/commands/compile/run_pipeline.rs index 8dee26ecab..a146350f3b 100644 --- a/crates/perry/src/commands/compile/run_pipeline.rs +++ b/crates/perry/src/commands/compile/run_pipeline.rs @@ -106,6 +106,32 @@ fn is_type_only_export_binding(module: &perry_hir::Module, name: &str) -> bool { !has_runtime_value } +/// Wrapper symbol used when a dynamic-import namespace materializes a local +/// function as a JavaScript value. Keep this routed through codegen's own +/// function mangler: module-name sanitization is intentionally not injective +/// for `$` and would point `$a` at `_a`'s symbol instead. +fn namespace_local_function_wrapper_symbol(module_name: &str, function_name: &str) -> String { + format!( + "__perry_wrap_{}", + perry_codegen::user_function_symbol(module_name, function_name) + ) +} + +#[cfg(test)] +mod namespace_local_function_symbol_tests { + use super::namespace_local_function_wrapper_symbol; + + #[test] + fn uses_injective_function_component_for_dynamic_namespace_entries() { + let dollar = namespace_local_function_wrapper_symbol("chunk.js", "$a"); + let underscore = namespace_local_function_wrapper_symbol("chunk.js", "_a"); + + assert_eq!(dollar, "__perry_wrap_perry_fn_chunk_js__u__24_a"); + assert_eq!(underscore, "__perry_wrap_perry_fn_chunk_js___a"); + assert_ne!(dollar, underscore); + } +} + // OpenCode's 0.5--1.0 MiB generated chunks routinely lower to 20--45 MiB of // LLVM input even with fewer than 1,000 HIR callables. Treat that observed // range as memory-heavy too: ordinary modules still use outer parallelism, @@ -2766,13 +2792,11 @@ pub fn run_with_parse_cache( .iter() .find(|f| f.name == fe.source_local) { - let scoped = format!( - "perry_fn_{}__{}", - sanitize_module_name(&target_hir.name), - sanitize_module_name(&func.name) - ); perry_codegen::NamespaceEntryKind::LocalFunction { - wrap_symbol: format!("__perry_wrap_{}", scoped), + wrap_symbol: namespace_local_function_wrapper_symbol( + &target_hir.name, + &func.name, + ), } } else if let Some(class) = target_hir .classes @@ -2807,13 +2831,11 @@ pub fn run_with_parse_cache( // ran during init → "Cannot read properties of undefined // (reading 'checks')"). Resolve to the ORIGIN function's // closure singleton instead, matching plain declarations. - let scoped = format!( - "perry_fn_{}__{}", - sanitize_module_name(&target_hir.name), - sanitize_module_name(&func.name) - ); perry_codegen::NamespaceEntryKind::LocalFunction { - wrap_symbol: format!("__perry_wrap_{}", scoped), + wrap_symbol: namespace_local_function_wrapper_symbol( + &target_hir.name, + &func.name, + ), } } else { // Best-effort: treat unknown locals as Var sourced diff --git a/docs/src/internals/garbage-collector.md b/docs/src/internals/garbage-collector.md index a8a2865dcd..670d013a95 100644 --- a/docs/src/internals/garbage-collector.md +++ b/docs/src/internals/garbage-collector.md @@ -255,7 +255,8 @@ These are the operational controls most useful outside collector development: | `PERRY_RS4GC=0` | select shadow roots on a native-root-capable target | | `PERRY_CONSERVATIVE_STACK_SCAN=full` | diagnostic full native-stack scan; disables copying | | `PERRY_GC_TRACE=1` | emit structured per-cycle trace records | -| `PERRY_GC_DIAG=1` | emit human-readable collector diagnostics | +| `PERRY_GC_DIAG=1` | emit human-readable collector diagnostics (per cycle, plus `[gc-trigger]`/`[gc-full]`/`[gc-budgeted]`/`[gc-charge]` decision and charge attribution and the per-minor `[gc-survival]` root attribution) | +| `PERRY_ALLOC_SITE_SAMPLE=N` | sample the arena allocation-site histogram every N bytes (`[alloc-site]`); `1`/`on` selects the default interval | Rooting stress uses `PERRY_GC_SCHEDULE_SEED`, `PERRY_GC_SCHEDULE_RATE`, `PERRY_GC_SCHEDULE_ALLOC_KB`, diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index b52c8629f5..b5898b2083 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -136,6 +136,12 @@ "verdict": "not_a_gc_pointer", "why": "#9771: bytes remaining to allocate before the next native-heap sample. A `Cell` counter, const-initialised so the TLS access itself never allocates." }, + { + "file": "crates/perry-runtime/src/arena/alloc_sample.rs", + "name": "UNTIL", + "verdict": "not_a_gc_pointer", + "why": "#9794 allocation-site sampling: bytes remaining until the next sample. A `Cell` countdown, decremented per allocation and reset on fire \u2014 a quantity, never an address." + }, { "file": "crates/perry-runtime/src/async_hooks.rs", "name": "ASYNC_HOOK_HANDLES", @@ -270,7 +276,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -285,10 +291,10 @@ "function": "run_to_completion" }, "sources": { - "crates/perry-runtime/src/gc/census.rs": "8050a9d1ca15f783195ccfa5963089b60bc4a6c31d9ced5755537623e70e7e3c", + "crates/perry-runtime/src/gc/census.rs": "388414f9629f196e84673e91bebd04bdcdcabdaa180252d2dfe4b82d1b49ca5a", "crates/perry-runtime/src/gc/cycle.rs": "763d552271b8e983a796b4e9648cd8ee984a0602b2b56aeefdb8713c0049c31f", - "crates/perry-runtime/src/gc/mod.rs": "085c3dcde34a172aa2b96ee4500658abae77cd34ee7f0e2dfeee06ae5774a414", - "crates/perry-runtime/src/gc/policy.rs": "319ed42f1a985c88f6362657a08518077283fe5216d6055fc82343b34dec50f9", + "crates/perry-runtime/src/gc/mod.rs": "7dd42b9506a97e6844fd3225dc53dfd59512631784750f58ff72208d68595481", + "crates/perry-runtime/src/gc/policy.rs": "fa8e9fa188d50bd92c3fbe23950a195906baf387f3a208497791bdd23f1c72db", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } @@ -305,12 +311,54 @@ "verdict": "test_only", "why": "Declared under cfg(test); holds an explicitly leaked Rust path string used by the isolated census unit tests." }, + { + "file": "crates/perry-runtime/src/gc/diag_sites.rs", + "name": "BUDGETED", + "verdict": "not_a_gc_pointer", + "why": "#9794 budgeted-cycle attribution: `BudgetedCycleDiag` is counters and byte/slice tallies for the cycle in progress. No address fields." + }, + { + "file": "crates/perry-runtime/src/gc/diag_sites.rs", + "name": "FULL_SITE", + "verdict": "not_a_gc_pointer", + "why": "#9794 trigger attribution: the `&'static str` name of the site that requested the current full collection. A pointer into rodata, not a GC allocation." + }, + { + "file": "crates/perry-runtime/src/gc/diag_sites.rs", + "name": "FULL_SITE_COUNTS", + "verdict": "not_a_gc_pointer", + "why": "#9794 trigger attribution: per-site full-collection tallies, `Vec<(&'static str, u32)>`. Both members are rodata string literals and counts; nothing the collector traces." + }, + { + "file": "crates/perry-runtime/src/gc/diag_sites.rs", + "name": "LAST_BUDGETED", + "verdict": "not_a_gc_pointer", + "why": "#9794: the previous budgeted cycle's four tallies as a `(u64, u64, u64, u64)`. Numbers only." + }, + { + "file": "crates/perry-runtime/src/gc/diag_sites.rs", + "name": "PRIMITIVE_DISPATCH", + "verdict": "not_a_gc_pointer", + "why": "#9794: per-method primitive-dispatch counts, `HashMap`. Rust-owned method-name strings and counters." + }, + { + "file": "crates/perry-runtime/src/gc/diag_sites.rs", + "name": "STRING_WRAPPERS", + "verdict": "not_a_gc_pointer", + "why": "#9794: materialized/elided string-wrapper counts as a `(u64, u64)` pair." + }, { "file": "crates/perry-runtime/src/gc/oldgen_defrag.rs", "name": "LAST_IDLE_PREDICTED_RELEASE", "verdict": "not_a_gc_pointer", "why": "#9772: releasable block BYTES the last idle selection promised \u2014 a size, not an address. A `Cell` compared against what the collection actually released." }, + { + "file": "crates/perry-runtime/src/gc/survival_diag.rs", + "name": "MINOR_SEQ", + "verdict": "not_a_gc_pointer", + "why": "#9794: monotonic per-minor sequence number used to label survival-origin records. A `Cell`." + }, { "file": "crates/perry-runtime/src/gc/trace.rs", "name": "FORWARDED_STUB_MEMBERSHIP_RECOVERIES", @@ -481,12 +529,6 @@ "verdict": "not_a_gc_pointer", "why": "First-insertion order for CLASS_DYNAMIC_PROPS: HashMap> of owned Rust strings, needed because the value table is a HashMap while [[OwnPropertyKeys]] needs order. Holds no JSValues; the f64 values live in CLASS_DYNAMIC_PROPS, which is already a scanned root." }, - { - "file": "crates/perry-runtime/src/object/mod.rs", - "name": "SHAPE_CACHE_YOUNG", - "verdict": "not_a_gc_pointer", - "why": "#9754 remembered set: `YoungLog` of shape-cache IDs (inline slot and overflow key alike) whose keys array a minor may act on. Ids, not addresses; the arrays themselves are visited by the registered shape-cache scanner." - }, { "file": "crates/perry-runtime/src/object/mod.rs", "name": "TRANSITION_CACHE_YOUNG", @@ -575,6 +617,12 @@ "verdict": "not_a_gc_pointer", "why": "Atomic count of live PTY handles that currently keep the event loop active. It stores only a scalar count; PTY JS values live in PTY_LIVE and are visited by pty_reactor_scan_roots_mut." }, + { + "file": "crates/perry-runtime/src/regex.rs", + "name": "NEVER_MATCH", + "verdict": "not_a_gc_pointer", + "why": "#9796: the memoized never-match placeholder program installed for a pattern only `fancy-regex` accepts. `Arc` is a Rust-allocator compiled program \u2014 the collector neither traces nor moves it \u2014 and the `Arc` keeps it alive independently." + }, { "file": "crates/perry-runtime/src/regex.rs", "name": "REGEX_POINTERS", diff --git a/scripts/raw_handle_debt_files.txt b/scripts/raw_handle_debt_files.txt index faf4abbbb0..f18eb47086 100644 --- a/scripts/raw_handle_debt_files.txt +++ b/scripts/raw_handle_debt_files.txt @@ -129,7 +129,7 @@ 3 crates/perry-runtime/src/promise/then.rs 8 crates/perry-runtime/src/proxy.rs 6 crates/perry-runtime/src/proxy/put_value.rs -6 crates/perry-runtime/src/regex.rs +2 crates/perry-runtime/src/regex.rs 19 crates/perry-runtime/src/regex/exec_array.rs 13 crates/perry-runtime/src/regex/match_all.rs 2 crates/perry-runtime/src/regex/match_string.rs diff --git a/scripts/shape_descriptor_census_baseline.json b/scripts/shape_descriptor_census_baseline.json index ce87ef8648..4317c19d7f 100644 --- a/scripts/shape_descriptor_census_baseline.json +++ b/scripts/shape_descriptor_census_baseline.json @@ -47,7 +47,6 @@ "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|keys_array: std::ptr::null_mut(),": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|pub(crate) fn test_seed_shape_cache_root(shape_id: u32, keys_array: *mut ArrayHeader) {": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|pub(crate) fn test_shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) {": 1, - "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|pub(super) fn arm_shape_cache_young(shape_id: u32, keys_array: *mut ArrayHeader) {": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|unsafe fn set_object_keys_array(obj: *mut ObjectHeader, keys_array: *mut ArrayHeader) {": 1, "crates/perry-runtime/src/object/object_ops.rs|keys_array|declaration|pub(crate) use keys_array::{": 1, "crates/perry-runtime/src/object/side_table_roots.rs|keys_array|access|visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array);": 1, @@ -57,7 +56,7 @@ "codegen_object_header_size_sites": 43, "raw_member_files": 9, "raw_member_sites": { - "keys_array": 26 + "keys_array": 25 } } } diff --git a/scripts/string_payload_access_baseline.txt b/scripts/string_payload_access_baseline.txt index cc8c57e8b0..b9817beaa0 100644 --- a/scripts/string_payload_access_baseline.txt +++ b/scripts/string_payload_access_baseline.txt @@ -12,7 +12,7 @@ inline-offset | perry-ext-nodemailer | 1 inline-offset | perry-ext-pg | 2 inline-offset | perry-ext-zlib | 3 inline-offset | perry-ffi | 3 -inline-offset | perry-runtime | 353 +inline-offset | perry-runtime | 354 inline-offset | perry-stdlib | 40 inline-offset | perry-updater | 5 reader-helper | perry-ext-ethers | 1 diff --git a/test-files/test_issue_9810_virtual_string_indices.ts b/test-files/test_issue_9810_virtual_string_indices.ts new file mode 100644 index 0000000000..4dca0332f6 --- /dev/null +++ b/test-files/test_issue_9810_virtual_string_indices.ts @@ -0,0 +1,119 @@ +// #9810: boxing must not eagerly allocate one property per UTF-16 code unit. +function check(ok: boolean, label: string): void { + if (!ok) throw new Error(label); +} +function equal(actual: any, expected: any, label: string): void { + check(JSON.stringify(actual) === JSON.stringify(expected), label); +} +function throws(fn: () => void, label: string): void { + let threw = false; + try { fn(); } catch (e) { threw = e instanceof TypeError; } + check(threw, label); +} + +const text = "a😀b"; +const s: any = Object(text); +check(s.length === 4 && s.valueOf() === text, "payload and UTF-16 length"); +for (let i = 0; i < 4; i++) { + check(s[i] === text[i], "index read " + i); + check(Object.hasOwn(s, String(i)) && s.hasOwnProperty(i) && i in s, "own index " + i); + check(s.propertyIsEnumerable(i), "enumerable index " + i); + equal(Object.getOwnPropertyDescriptor(s, String(i)), { + value: text[i], writable: false, enumerable: true, configurable: false, + }, "index descriptor " + i); +} +for (const key of ["-0", "-1", "01", "1.0", "1.5", "4", "NaN", "4294967295"]) { + check(!Object.hasOwn(s, key), "absent " + key); +} +const symbol = Symbol("extra"); +s[symbol] = 17; +s.extra = 9; +s[7] = "outside"; +s["01"] = "leading"; +Object.defineProperty(s, "hidden", { value: 10 }); +equal(Object.keys(s), ["0", "1", "2", "3", "7", "extra", "01"], "keys order"); +equal(Object.getOwnPropertyNames(s), ["0", "1", "2", "3", "7", "length", "extra", "01", "hidden"], "names order"); +const ownKeys = Reflect.ownKeys(s); +check(ownKeys.length === 10 && ownKeys[9] === symbol, "symbol ordering"); +equal(Object.values(s), [text[0], text[1], text[2], text[3], "outside", 9, "leading"], "values"); +equal(Object.entries(Object("ab")), [["0", "a"], ["1", "b"]], "entries"); +equal(Object.keys(Object.getOwnPropertyDescriptors(Object("ab"))), ["0", "1", "length"], "descriptors"); +const loop: string[] = []; +for (const key in s) loop.push(key); +equal(loop, Object.keys(s), "for-in"); +const assigned: any = Object.assign({}, s); +check(assigned[0] === "a" && assigned[3] === "b" && assigned.extra === 9 && assigned[symbol] === 17, "assign"); +check(!Object.hasOwn(assigned, "length") && !Object.hasOwn(assigned, "hidden"), "assign filters"); +const spread: any = { ...s, tail: 12 }; +check(spread[0] === "a" && spread[3] === "b" && spread.tail === 12, "spread"); +const { 0: first, ...rest } = s; +check(first === "a" && !Object.hasOwn(rest, "0") && rest[3] === "b", "rest"); +check(JSON.stringify(s) === JSON.stringify(text), "JSON unwraps"); + +check(!Reflect.set(s, "0", "x"), "Reflect.set rejects"); +check(!Reflect.deleteProperty(s, "0"), "Reflect.delete rejects"); +check(!Reflect.defineProperty(s, "0", { value: "x" }), "Reflect.define rejects"); +Object.defineProperty(s, "0", { value: "a" }); +Object.defineProperty(s, "0", {}); +Object.defineProperty(s, "0", { writable: false, enumerable: true, configurable: false }); +throws(() => Object.defineProperty(s, "0", { writable: true }), "cannot become writable"); +throws(() => Object.defineProperty(s, "0", { enumerable: false }), "cannot hide"); +throws(() => Object.defineProperty(s, "0", { configurable: true }), "cannot become configurable"); +throws(() => Object.defineProperty(s, "0", { get() { return "a"; } }), "cannot become accessor"); +throws(() => { "use strict"; s[0] = "x"; }, "strict assignment"); +throws(() => { "use strict"; delete s[0]; }, "strict delete"); +check(s[0] === "a", "rejected operations preserve index"); +check(delete s[7] && !Object.hasOwn(s, "7"), "delete expando"); +Object.preventExtensions(s); +Object.defineProperty(s, "0", { value: "a" }); +check(!Reflect.defineProperty(s, "8", { value: "new" }), "no new property after preventExtensions"); +for (const lock of [Object.seal, Object.freeze]) { + const locked: any = lock(Object("ab")); + check(Object.isSealed(locked) && Object.isFrozen(locked), "immutable sealed indices"); + equal(Object.keys(locked), ["0", "1"], "locked enumeration"); + check(!Reflect.deleteProperty(locked, "1"), "locked delete"); +} + +// Own virtual indices must shadow inherited numeric accessors/properties. +const proto: any = { 0: "wrong", inherited: 1 }; +const changed: any = Object("ab"); +Object.setPrototypeOf(changed, proto); +check(changed[0] === "a" && changed.inherited === 1, "custom prototype"); +Object.defineProperty(proto, "1", { get() { return "wrong"; } }); +check(changed[1] === "b", "own index shadows inherited getter"); +Object.setPrototypeOf(changed, null); +check(changed[0] === "a" && Object.hasOwn(changed, "1"), "null prototype"); + +// Wide expando objects use a separate ownership index; its miss is not proof +// that a virtual character property is absent. +const wide: any = Object("abc"); +for (let i = 0; i < 80; i++) wide["field" + i] = i; +check(Object.hasOwn(wide, "1"), "wide own index"); +Object.defineProperty(wide, "1", { value: "b" }); +throws(() => Object.defineProperty(wide, "1", { value: "x" }), "wide incompatible definition"); + +const changing: any = Object("ab"); +Object.defineProperty(changing, "first", { enumerable: true, get() { + delete changing.later; + Object.defineProperty(changing, "hiddenLater", { enumerable: false }); + return 5; +} }); +changing.later = 6; +changing.hiddenLater = 7; +equal(Object.values(changing), ["a", "b", 5], "getter changes later keys"); + +function capture() { return Object(this); } +const methods: any = { capture }; +const a: any = methods.capture.call("x".repeat(200)); +const b: any = methods.capture.apply("x".repeat(200), []); +check(typeof a === "object" && a !== b && a.length === 200 && b[199] === "x", "call/apply wrappers"); +a.extra = 4; +check(b.extra === undefined, "independent receiver state"); +function strictReceiver() { "use strict"; return typeof this; } +check(strictReceiver.call("abc") === "string", "strict primitive receiver"); +(String.prototype as any).issue9810 = capture; +const methodThis: any = ("abc" as any).issue9810(); +check(typeof methodThis === "object" && methodThis[2] === "c", "prototype method receiver"); +delete (String.prototype as any).issue9810; +equal(Object.keys(Object("")), [], "empty wrapper"); +console.log("virtual-string-indices-9810 ok");