From 5ad8ce40bb7ff5116ff39f8550de35db7230f5ce Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 26 May 2026 14:09:48 -0700 Subject: [PATCH 1/4] Don't panic in `Slab::{get,get_mut}` Instead return `None`. This helps prevent corruption of the GC heap from causing panics. --- crates/core/src/slab.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/crates/core/src/slab.rs b/crates/core/src/slab.rs index 38a483a839f9..030d6294a2ea 100644 --- a/crates/core/src/slab.rs +++ b/crates/core/src/slab.rs @@ -390,15 +390,11 @@ impl Slab { /// /// Returns `None` if the value has since been deallocated. /// - /// If `id` comes from a different `Slab` instance, this method may panic, - /// return `None`, or return an arbitrary value. + /// If `id` comes from a different `Slab` instance, this method may return + /// `None`, or return an arbitrary value. #[inline] pub fn get(&self, id: Id) -> Option<&T> { - match self - .entries - .get(id.0.index()) - .expect("id from different slab") - { + match self.entries.get(id.0.index())? { Entry::Occupied(x) => Some(x), Entry::Free { .. } => None, } @@ -408,15 +404,11 @@ impl Slab { /// /// Returns `None` if the value has since been deallocated. /// - /// If `id` comes from a different `Slab` instance, this method may panic, - /// return `None`, or return an arbitrary value. + /// If `id` comes from a different `Slab` instance, this method may return + /// `None`, or return an arbitrary value. #[inline] pub fn get_mut(&mut self, id: Id) -> Option<&mut T> { - match self - .entries - .get_mut(id.0.index()) - .expect("id from different slab") - { + match self.entries.get_mut(id.0.index())? { Entry::Occupied(x) => Some(x), Entry::Free { .. } => None, } From 5c4905dce5113aeb7a0d018846f154e455d41b3c Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 26 May 2026 14:14:08 -0700 Subject: [PATCH 2/4] Don't allow memset with funcref arrays Even `ref.null func` is intern'd, so nothing is candidate for memset on a `funcref` array. Overall just ban memset on all reference-typed arrays for now to optimize the easy case of scalar data, but leave optimizing reference types for later. --- crates/cranelift/src/func_environ.rs | 11 +- tests/all/fuel.wast | 4 +- tests/disas/array-fill-anyref.wat | 38 ++++--- tests/disas/array-fill-externref.wat | 38 ++++--- tests/disas/array-fill-funcref.wat | 43 +++++--- tests/disas/array-fill-i31ref.wat | 38 ++++--- tests/disas/gc/array-new-default-anyref.wat | 103 ------------------ tests/disas/gc/array-new-default-exnref.wat | 103 ------------------ .../disas/gc/array-new-default-externref.wat | 103 ------------------ tests/disas/gc/array-new-default-funcref.wat | 90 ++------------- .../gc/array-fill-null-funcref.wast | 10 ++ 11 files changed, 134 insertions(+), 447 deletions(-) create mode 100644 tests/misc_testsuite/gc/array-fill-null-funcref.wast diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index b3568e4070a4..e6e78366c49f 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -5798,8 +5798,15 @@ impl CheckedEntity { /// bit patterns. fn allows_memset(&self, env: &FuncEnvironment) -> bool { match self { - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } | CheckedEntity::Array { .. } => { - true + CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => true, + CheckedEntity::Array { ty, .. } => { + let array_ty = env.types.unwrap_array(*ty).unwrap(); + // Most GC references need barriers, funcrefs need intern-ing, + // etc. Disallow memset on all reference types. + !matches!( + array_ty.0.element_type, + WasmStorageType::Val(WasmValType::Ref(_)) + ) } CheckedEntity::Elem(_) => false, // Tables that are lazily initialized can't be memset because the diff --git a/tests/all/fuel.wast b/tests/all/fuel.wast index 7196df646a94..46613616661f 100644 --- a/tests/all/fuel.wast +++ b/tests/all/fuel.wast @@ -365,7 +365,7 @@ (start $f) (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f))) -(assert_fuel 403 ;; 400 bytes set + 1 func + 2 instructions (drop is 0) +(assert_fuel 103 (module (type $a (array (mut funcref))) (func $f @@ -375,7 +375,7 @@ ) (start $f))) -(assert_fuel 404 ;; 400 bytes set + 1 func + 3 instructions (drop is 0) +(assert_fuel 104 (module (type $a (array (mut funcref))) (func $f diff --git a/tests/disas/array-fill-anyref.wat b/tests/disas/array-fill-anyref.wat index 52bc52418829..428cb320f919 100644 --- a/tests/disas/array-fill-anyref.wat +++ b/tests/disas/array-fill-anyref.wat @@ -81,14 +81,12 @@ ;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:2 sig0 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): ;; @003e trapz v2, user16 -;; @003e v44 = load.i64 notrap aligned readonly can_move v0+8 -;; @003e v7 = load.i64 notrap aligned readonly can_move v44+32 +;; @003e v49 = load.i64 notrap aligned readonly can_move v0+8 +;; @003e v7 = load.i64 notrap aligned readonly can_move v49+32 ;; @003e v6 = uextend.i64 v2 ;; @003e v8 = iadd v7, v6 ;; @003e v9 = iconst.i64 16 @@ -100,19 +98,33 @@ ;; @003e v12 = uextend.i64 v11 ;; @003e v17 = icmp ugt v16, v12 ;; @003e trapnz v17, user17 -;; @003e v28 = load.i64 notrap aligned v44+40 -;; v40 = iconst.i64 20 -;; @003e v21 = iadd v8, v40 ; v40 = 20 -;; v48 = iconst.i64 2 -;; v49 = ishl v13, v48 ; v48 = 2 -;; @003e v24 = iadd v21, v49 -;; v51 = ishl v14, v48 ; v48 = 2 -;; @003e v30 = uadd_overflow_trap v24, v51, user2 +;; @003e v28 = load.i64 notrap aligned v49+40 +;; v45 = iconst.i64 20 +;; @003e v21 = iadd v8, v45 ; v45 = 20 +;; v53 = iconst.i64 2 +;; v54 = ishl v13, v53 ; v53 = 2 +;; @003e v24 = iadd v21, v54 +;; v56 = ishl v14, v53 ; v53 = 2 +;; @003e v30 = uadd_overflow_trap v24, v56, user2 ;; @003e v29 = iadd v7, v28 ;; @003e v31 = icmp ugt v30, v29 ;; @003e trapnz v31, user2 +;; v51 = iconst.i64 0 +;; @003e v33 = icmp eq v14, v51 ; v51 = 0 ;; @003a v5 = iconst.i32 0 -;; @003e call fn0(v0, v24, v5, v51) ; v5 = 0 +;; v44 = iconst.i64 4 +;; @003e v32 = iadd v24, v56 +;; @003e brif v33, block3, block2(v24) +;; +;; block2(v34: i64): +;; v58 = iconst.i32 0 +;; @003e store user2 little v58, v34 ; v58 = 0 +;; v59 = iconst.i64 4 +;; v60 = iadd v34, v59 ; v59 = 4 +;; @003e v36 = icmp eq v60, v32 +;; @003e brif v36, block3, block2(v60) +;; +;; block3: ;; @0041 jump block1 ;; ;; block1: diff --git a/tests/disas/array-fill-externref.wat b/tests/disas/array-fill-externref.wat index a327322edef5..542857bb457a 100644 --- a/tests/disas/array-fill-externref.wat +++ b/tests/disas/array-fill-externref.wat @@ -77,14 +77,12 @@ ;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:2 sig0 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): ;; @003d trapz v2, user16 -;; @003d v44 = load.i64 notrap aligned readonly can_move v0+8 -;; @003d v7 = load.i64 notrap aligned readonly can_move v44+32 +;; @003d v49 = load.i64 notrap aligned readonly can_move v0+8 +;; @003d v7 = load.i64 notrap aligned readonly can_move v49+32 ;; @003d v6 = uextend.i64 v2 ;; @003d v8 = iadd v7, v6 ;; @003d v9 = iconst.i64 16 @@ -96,19 +94,33 @@ ;; @003d v12 = uextend.i64 v11 ;; @003d v17 = icmp ugt v16, v12 ;; @003d trapnz v17, user17 -;; @003d v28 = load.i64 notrap aligned v44+40 -;; v40 = iconst.i64 20 -;; @003d v21 = iadd v8, v40 ; v40 = 20 -;; v48 = iconst.i64 2 -;; v49 = ishl v13, v48 ; v48 = 2 -;; @003d v24 = iadd v21, v49 -;; v51 = ishl v14, v48 ; v48 = 2 -;; @003d v30 = uadd_overflow_trap v24, v51, user2 +;; @003d v28 = load.i64 notrap aligned v49+40 +;; v45 = iconst.i64 20 +;; @003d v21 = iadd v8, v45 ; v45 = 20 +;; v53 = iconst.i64 2 +;; v54 = ishl v13, v53 ; v53 = 2 +;; @003d v24 = iadd v21, v54 +;; v56 = ishl v14, v53 ; v53 = 2 +;; @003d v30 = uadd_overflow_trap v24, v56, user2 ;; @003d v29 = iadd v7, v28 ;; @003d v31 = icmp ugt v30, v29 ;; @003d trapnz v31, user2 +;; v51 = iconst.i64 0 +;; @003d v33 = icmp eq v14, v51 ; v51 = 0 ;; @0039 v5 = iconst.i32 0 -;; @003d call fn0(v0, v24, v5, v51) ; v5 = 0 +;; v44 = iconst.i64 4 +;; @003d v32 = iadd v24, v56 +;; @003d brif v33, block3, block2(v24) +;; +;; block2(v34: i64): +;; v58 = iconst.i32 0 +;; @003d store user2 little v58, v34 ; v58 = 0 +;; v59 = iconst.i64 4 +;; v60 = iadd v34, v59 ; v59 = 4 +;; @003d v36 = icmp eq v60, v32 +;; @003d brif v36, block3, block2(v60) +;; +;; block3: ;; @0040 jump block1 ;; ;; block1: diff --git a/tests/disas/array-fill-funcref.wat b/tests/disas/array-fill-funcref.wat index 4f594096bb1c..026b75494258 100644 --- a/tests/disas/array-fill-funcref.wat +++ b/tests/disas/array-fill-funcref.wat @@ -88,14 +88,14 @@ ;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:2 sig0 +;; sig0 = (i64 vmctx, i64) -> i64 tail +;; fn0 = colocated u805306368:25 sig0 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): ;; @0049 trapz v2, user16 -;; @0049 v44 = load.i64 notrap aligned readonly can_move v0+8 -;; @0049 v7 = load.i64 notrap aligned readonly can_move v44+32 +;; @0049 v52 = load.i64 notrap aligned readonly can_move v0+8 +;; @0049 v7 = load.i64 notrap aligned readonly can_move v52+32 ;; @0049 v6 = uextend.i64 v2 ;; @0049 v8 = iadd v7, v6 ;; @0049 v9 = iconst.i64 16 @@ -107,19 +107,34 @@ ;; @0049 v12 = uextend.i64 v11 ;; @0049 v17 = icmp ugt v16, v12 ;; @0049 trapnz v17, user17 -;; @0049 v28 = load.i64 notrap aligned v44+40 -;; v40 = iconst.i64 20 -;; @0049 v21 = iadd v8, v40 ; v40 = 20 -;; v47 = iconst.i64 2 -;; v48 = ishl v13, v47 ; v47 = 2 -;; @0049 v24 = iadd v21, v48 -;; v50 = ishl v14, v47 ; v47 = 2 -;; @0049 v30 = uadd_overflow_trap v24, v50, user2 +;; @0049 v28 = load.i64 notrap aligned v52+40 +;; v48 = iconst.i64 20 +;; @0049 v21 = iadd v8, v48 ; v48 = 20 +;; v55 = iconst.i64 2 +;; v56 = ishl v13, v55 ; v55 = 2 +;; @0049 v24 = iadd v21, v56 +;; v58 = ishl v14, v55 ; v55 = 2 +;; @0049 v30 = uadd_overflow_trap v24, v58, user2 ;; @0049 v29 = iadd v7, v28 ;; @0049 v31 = icmp ugt v30, v29 ;; @0049 trapnz v31, user2 -;; @0049 v32 = iconst.i32 0 -;; @0049 call fn0(v0, v24, v32, v50) ; v32 = 0 +;; @0045 v5 = iconst.i64 0 +;; @0049 v33 = icmp eq v14, v5 ; v5 = 0 +;; v47 = iconst.i64 4 +;; @0049 v32 = iadd v24, v58 +;; @0049 brif v33, block3, block2(v24) +;; +;; block2(v34: i64): +;; v60 = iconst.i64 0 +;; @0049 v36 = call fn0(v0, v60) ; v60 = 0 +;; @0049 v37 = ireduce.i32 v36 +;; @0049 store user2 little v37, v34 +;; v61 = iconst.i64 4 +;; v62 = iadd v34, v61 ; v61 = 4 +;; @0049 v39 = icmp eq v62, v32 +;; @0049 brif v39, block3, block2(v62) +;; +;; block3: ;; @004c jump block1 ;; ;; block1: diff --git a/tests/disas/array-fill-i31ref.wat b/tests/disas/array-fill-i31ref.wat index 6723ac376d17..b094bdde3f99 100644 --- a/tests/disas/array-fill-i31ref.wat +++ b/tests/disas/array-fill-i31ref.wat @@ -81,14 +81,12 @@ ;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:2 sig0 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): ;; @003e trapz v2, user16 -;; @003e v44 = load.i64 notrap aligned readonly can_move v0+8 -;; @003e v7 = load.i64 notrap aligned readonly can_move v44+32 +;; @003e v49 = load.i64 notrap aligned readonly can_move v0+8 +;; @003e v7 = load.i64 notrap aligned readonly can_move v49+32 ;; @003e v6 = uextend.i64 v2 ;; @003e v8 = iadd v7, v6 ;; @003e v9 = iconst.i64 16 @@ -100,19 +98,33 @@ ;; @003e v12 = uextend.i64 v11 ;; @003e v17 = icmp ugt v16, v12 ;; @003e trapnz v17, user17 -;; @003e v28 = load.i64 notrap aligned v44+40 -;; v40 = iconst.i64 20 -;; @003e v21 = iadd v8, v40 ; v40 = 20 -;; v48 = iconst.i64 2 -;; v49 = ishl v13, v48 ; v48 = 2 -;; @003e v24 = iadd v21, v49 -;; v51 = ishl v14, v48 ; v48 = 2 -;; @003e v30 = uadd_overflow_trap v24, v51, user2 +;; @003e v28 = load.i64 notrap aligned v49+40 +;; v45 = iconst.i64 20 +;; @003e v21 = iadd v8, v45 ; v45 = 20 +;; v53 = iconst.i64 2 +;; v54 = ishl v13, v53 ; v53 = 2 +;; @003e v24 = iadd v21, v54 +;; v56 = ishl v14, v53 ; v53 = 2 +;; @003e v30 = uadd_overflow_trap v24, v56, user2 ;; @003e v29 = iadd v7, v28 ;; @003e v31 = icmp ugt v30, v29 ;; @003e trapnz v31, user2 +;; v51 = iconst.i64 0 +;; @003e v33 = icmp eq v14, v51 ; v51 = 0 ;; @003a v5 = iconst.i32 0 -;; @003e call fn0(v0, v24, v5, v51) ; v5 = 0 +;; v44 = iconst.i64 4 +;; @003e v32 = iadd v24, v56 +;; @003e brif v33, block3, block2(v24) +;; +;; block2(v34: i64): +;; v58 = iconst.i32 0 +;; @003e store user2 little v58, v34 ; v58 = 0 +;; v59 = iconst.i64 4 +;; v60 = iadd v34, v59 ; v59 = 4 +;; @003e v36 = icmp eq v60, v32 +;; @003e brif v36, block3, block2(v60) +;; +;; block3: ;; @0041 jump block1 ;; ;; block1: diff --git a/tests/disas/gc/array-new-default-anyref.wat b/tests/disas/gc/array-new-default-anyref.wat index 82015ac7691f..cccc027faadf 100644 --- a/tests/disas/gc/array-new-default-anyref.wat +++ b/tests/disas/gc/array-new-default-anyref.wat @@ -10,106 +10,3 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { -;; ss0 = explicit_slot 4, align = 4 -;; region0 = 2 "vmctx" -;; gv0 = vmctx -;; gv1 = load.i64 notrap aligned readonly gv0+8 -;; gv2 = load.i64 notrap aligned gv1+24 -;; gv3 = vmctx -;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 -;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 -;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail -;; sig1 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:24 sig0 -;; fn1 = colocated u805306368:2 sig1 -;; stack_limit = gv2 -;; -;; block0(v0: i64, v1: i64, v2: i32): -;; @001f v5 = uextend.i64 v2 -;; v102 = iconst.i64 2 -;; v103 = ishl v5, v102 ; v102 = 2 -;; v100 = iconst.i64 32 -;; @001f v7 = ushr v103, v100 ; v100 = 32 -;; @001f trapnz v7, user18 -;; @001f v4 = iconst.i32 20 -;; v109 = iconst.i32 2 -;; v110 = ishl v2, v109 ; v109 = 2 -;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20 -;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 -;; @001f v12 = load.i32 notrap aligned v11 -;; @001f v13 = load.i32 notrap aligned v11+4 -;; @001f v19 = uextend.i64 v12 -;; @001f v14 = uextend.i64 v9 -;; @001f v15 = iconst.i64 15 -;; @001f v17 = iadd v14, v15 ; v15 = 15 -;; @001f v16 = iconst.i64 -16 -;; @001f v18 = band v17, v16 ; v16 = -16 -;; @001f v20 = iadd v19, v18 -;; @001f v21 = uextend.i64 v13 -;; @001f v22 = icmp ule v20, v21 -;; @001f brif v22, block2, block3 -;; -;; block2: -;; v118 = iconst.i32 15 -;; v119 = iadd.i32 v9, v118 ; v118 = 15 -;; v122 = iconst.i32 -16 -;; v123 = band v119, v122 ; v122 = -16 -;; v125 = iadd.i32 v12, v123 -;; @001f store notrap aligned region0 v125, v11 -;; v141 = iconst.i32 -1476394994 -;; v142 = load.i64 notrap aligned readonly can_move v0+8 -;; v143 = load.i64 notrap aligned readonly can_move v142+32 -;; @001f v36 = iadd v143, v19 -;; @001f store notrap aligned v141, v36 ; v141 = -1476394994 -;; v144 = load.i64 notrap aligned readonly can_move v0+40 -;; v145 = load.i32 notrap aligned readonly can_move v144 -;; @001f store notrap aligned v145, v36+4 -;; v146 = band.i64 v17, v16 ; v16 = -16 -;; @001f istore32 notrap aligned v146, v36+8 -;; @001f jump block4(v12, v36) -;; -;; block3 cold: -;; @001f v24 = iconst.i32 -1476394994 -;; @001f v26 = load.i64 notrap aligned readonly can_move v0+40 -;; @001f v27 = load.i32 notrap aligned readonly can_move v26 -;; @001f v28 = iconst.i32 16 -;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 -;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8 -;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32 -;; @001f v31 = uextend.i64 v29 -;; @001f v32 = iadd v30, v31 -;; @001f jump block4(v29, v32) -;; -;; block4(v41: i32, v42: i64): -;; v95 = stack_addr.i64 ss0 -;; store notrap v41, v95 -;; v94 = iconst.i64 16 -;; @001f v43 = iadd v42, v94 ; v94 = 16 -;; @001f store.i32 user2 v2, v43 -;; v77 = load.i32 notrap v95 -;; @001f trapz v77, user16 -;; v147 = load.i64 notrap aligned readonly can_move v0+8 -;; v148 = load.i64 notrap aligned readonly can_move v147+32 -;; @001f v46 = uextend.i64 v77 -;; @001f v48 = iadd v148, v46 -;; @001f v50 = iadd v48, v94 ; v94 = 16 -;; @001f v51 = load.i32 user2 readonly v50 -;; @001f v52 = uextend.i64 v51 -;; @001f v57 = icmp.i64 ugt v5, v52 -;; @001f trapnz v57, user17 -;; @001f v68 = load.i64 notrap aligned v147+40 -;; v85 = iconst.i64 20 -;; @001f v61 = iadd v48, v85 ; v85 = 20 -;; @001f v70 = uadd_overflow_trap v61, v103, user2 -;; @001f v69 = iadd v148, v68 -;; @001f v71 = icmp ugt v70, v69 -;; @001f trapnz v71, user2 -;; @001f v44 = iconst.i32 0 -;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0 -;; v74 = load.i32 notrap v95 -;; @0022 jump block1 -;; -;; block1: -;; @0022 return v74 -;; } diff --git a/tests/disas/gc/array-new-default-exnref.wat b/tests/disas/gc/array-new-default-exnref.wat index 910d21579e02..ee921a2c8709 100644 --- a/tests/disas/gc/array-new-default-exnref.wat +++ b/tests/disas/gc/array-new-default-exnref.wat @@ -10,106 +10,3 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { -;; ss0 = explicit_slot 4, align = 4 -;; region0 = 2 "vmctx" -;; gv0 = vmctx -;; gv1 = load.i64 notrap aligned readonly gv0+8 -;; gv2 = load.i64 notrap aligned gv1+24 -;; gv3 = vmctx -;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 -;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 -;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail -;; sig1 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:24 sig0 -;; fn1 = colocated u805306368:2 sig1 -;; stack_limit = gv2 -;; -;; block0(v0: i64, v1: i64, v2: i32): -;; @001f v5 = uextend.i64 v2 -;; v102 = iconst.i64 2 -;; v103 = ishl v5, v102 ; v102 = 2 -;; v100 = iconst.i64 32 -;; @001f v7 = ushr v103, v100 ; v100 = 32 -;; @001f trapnz v7, user18 -;; @001f v4 = iconst.i32 20 -;; v109 = iconst.i32 2 -;; v110 = ishl v2, v109 ; v109 = 2 -;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20 -;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 -;; @001f v12 = load.i32 notrap aligned v11 -;; @001f v13 = load.i32 notrap aligned v11+4 -;; @001f v19 = uextend.i64 v12 -;; @001f v14 = uextend.i64 v9 -;; @001f v15 = iconst.i64 15 -;; @001f v17 = iadd v14, v15 ; v15 = 15 -;; @001f v16 = iconst.i64 -16 -;; @001f v18 = band v17, v16 ; v16 = -16 -;; @001f v20 = iadd v19, v18 -;; @001f v21 = uextend.i64 v13 -;; @001f v22 = icmp ule v20, v21 -;; @001f brif v22, block2, block3 -;; -;; block2: -;; v118 = iconst.i32 15 -;; v119 = iadd.i32 v9, v118 ; v118 = 15 -;; v122 = iconst.i32 -16 -;; v123 = band v119, v122 ; v122 = -16 -;; v125 = iadd.i32 v12, v123 -;; @001f store notrap aligned region0 v125, v11 -;; v141 = iconst.i32 -1476394994 -;; v142 = load.i64 notrap aligned readonly can_move v0+8 -;; v143 = load.i64 notrap aligned readonly can_move v142+32 -;; @001f v36 = iadd v143, v19 -;; @001f store notrap aligned v141, v36 ; v141 = -1476394994 -;; v144 = load.i64 notrap aligned readonly can_move v0+40 -;; v145 = load.i32 notrap aligned readonly can_move v144 -;; @001f store notrap aligned v145, v36+4 -;; v146 = band.i64 v17, v16 ; v16 = -16 -;; @001f istore32 notrap aligned v146, v36+8 -;; @001f jump block4(v12, v36) -;; -;; block3 cold: -;; @001f v24 = iconst.i32 -1476394994 -;; @001f v26 = load.i64 notrap aligned readonly can_move v0+40 -;; @001f v27 = load.i32 notrap aligned readonly can_move v26 -;; @001f v28 = iconst.i32 16 -;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 -;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8 -;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32 -;; @001f v31 = uextend.i64 v29 -;; @001f v32 = iadd v30, v31 -;; @001f jump block4(v29, v32) -;; -;; block4(v41: i32, v42: i64): -;; v95 = stack_addr.i64 ss0 -;; store notrap v41, v95 -;; v94 = iconst.i64 16 -;; @001f v43 = iadd v42, v94 ; v94 = 16 -;; @001f store.i32 user2 v2, v43 -;; v77 = load.i32 notrap v95 -;; @001f trapz v77, user16 -;; v147 = load.i64 notrap aligned readonly can_move v0+8 -;; v148 = load.i64 notrap aligned readonly can_move v147+32 -;; @001f v46 = uextend.i64 v77 -;; @001f v48 = iadd v148, v46 -;; @001f v50 = iadd v48, v94 ; v94 = 16 -;; @001f v51 = load.i32 user2 readonly v50 -;; @001f v52 = uextend.i64 v51 -;; @001f v57 = icmp.i64 ugt v5, v52 -;; @001f trapnz v57, user17 -;; @001f v68 = load.i64 notrap aligned v147+40 -;; v85 = iconst.i64 20 -;; @001f v61 = iadd v48, v85 ; v85 = 20 -;; @001f v70 = uadd_overflow_trap v61, v103, user2 -;; @001f v69 = iadd v148, v68 -;; @001f v71 = icmp ugt v70, v69 -;; @001f trapnz v71, user2 -;; @001f v44 = iconst.i32 0 -;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0 -;; v74 = load.i32 notrap v95 -;; @0022 jump block1 -;; -;; block1: -;; @0022 return v74 -;; } diff --git a/tests/disas/gc/array-new-default-externref.wat b/tests/disas/gc/array-new-default-externref.wat index 4a0771b172c0..4bad3865c1f5 100644 --- a/tests/disas/gc/array-new-default-externref.wat +++ b/tests/disas/gc/array-new-default-externref.wat @@ -10,106 +10,3 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { -;; ss0 = explicit_slot 4, align = 4 -;; region0 = 2 "vmctx" -;; gv0 = vmctx -;; gv1 = load.i64 notrap aligned readonly gv0+8 -;; gv2 = load.i64 notrap aligned gv1+24 -;; gv3 = vmctx -;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 -;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 -;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail -;; sig1 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:24 sig0 -;; fn1 = colocated u805306368:2 sig1 -;; stack_limit = gv2 -;; -;; block0(v0: i64, v1: i64, v2: i32): -;; @001f v5 = uextend.i64 v2 -;; v102 = iconst.i64 2 -;; v103 = ishl v5, v102 ; v102 = 2 -;; v100 = iconst.i64 32 -;; @001f v7 = ushr v103, v100 ; v100 = 32 -;; @001f trapnz v7, user18 -;; @001f v4 = iconst.i32 20 -;; v109 = iconst.i32 2 -;; v110 = ishl v2, v109 ; v109 = 2 -;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20 -;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 -;; @001f v12 = load.i32 notrap aligned v11 -;; @001f v13 = load.i32 notrap aligned v11+4 -;; @001f v19 = uextend.i64 v12 -;; @001f v14 = uextend.i64 v9 -;; @001f v15 = iconst.i64 15 -;; @001f v17 = iadd v14, v15 ; v15 = 15 -;; @001f v16 = iconst.i64 -16 -;; @001f v18 = band v17, v16 ; v16 = -16 -;; @001f v20 = iadd v19, v18 -;; @001f v21 = uextend.i64 v13 -;; @001f v22 = icmp ule v20, v21 -;; @001f brif v22, block2, block3 -;; -;; block2: -;; v118 = iconst.i32 15 -;; v119 = iadd.i32 v9, v118 ; v118 = 15 -;; v122 = iconst.i32 -16 -;; v123 = band v119, v122 ; v122 = -16 -;; v125 = iadd.i32 v12, v123 -;; @001f store notrap aligned region0 v125, v11 -;; v141 = iconst.i32 -1476394994 -;; v142 = load.i64 notrap aligned readonly can_move v0+8 -;; v143 = load.i64 notrap aligned readonly can_move v142+32 -;; @001f v36 = iadd v143, v19 -;; @001f store notrap aligned v141, v36 ; v141 = -1476394994 -;; v144 = load.i64 notrap aligned readonly can_move v0+40 -;; v145 = load.i32 notrap aligned readonly can_move v144 -;; @001f store notrap aligned v145, v36+4 -;; v146 = band.i64 v17, v16 ; v16 = -16 -;; @001f istore32 notrap aligned v146, v36+8 -;; @001f jump block4(v12, v36) -;; -;; block3 cold: -;; @001f v24 = iconst.i32 -1476394994 -;; @001f v26 = load.i64 notrap aligned readonly can_move v0+40 -;; @001f v27 = load.i32 notrap aligned readonly can_move v26 -;; @001f v28 = iconst.i32 16 -;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 -;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8 -;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32 -;; @001f v31 = uextend.i64 v29 -;; @001f v32 = iadd v30, v31 -;; @001f jump block4(v29, v32) -;; -;; block4(v41: i32, v42: i64): -;; v95 = stack_addr.i64 ss0 -;; store notrap v41, v95 -;; v94 = iconst.i64 16 -;; @001f v43 = iadd v42, v94 ; v94 = 16 -;; @001f store.i32 user2 v2, v43 -;; v77 = load.i32 notrap v95 -;; @001f trapz v77, user16 -;; v147 = load.i64 notrap aligned readonly can_move v0+8 -;; v148 = load.i64 notrap aligned readonly can_move v147+32 -;; @001f v46 = uextend.i64 v77 -;; @001f v48 = iadd v148, v46 -;; @001f v50 = iadd v48, v94 ; v94 = 16 -;; @001f v51 = load.i32 user2 readonly v50 -;; @001f v52 = uextend.i64 v51 -;; @001f v57 = icmp.i64 ugt v5, v52 -;; @001f trapnz v57, user17 -;; @001f v68 = load.i64 notrap aligned v147+40 -;; v85 = iconst.i64 20 -;; @001f v61 = iadd v48, v85 ; v85 = 20 -;; @001f v70 = uadd_overflow_trap v61, v103, user2 -;; @001f v69 = iadd v148, v68 -;; @001f v71 = icmp ugt v70, v69 -;; @001f trapnz v71, user2 -;; @001f v44 = iconst.i32 0 -;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0 -;; v74 = load.i32 notrap v95 -;; @0022 jump block1 -;; -;; block1: -;; @0022 return v74 -;; } diff --git a/tests/disas/gc/array-new-default-funcref.wat b/tests/disas/gc/array-new-default-funcref.wat index baa936d1ddfd..bb794de1c79b 100644 --- a/tests/disas/gc/array-new-default-funcref.wat +++ b/tests/disas/gc/array-new-default-funcref.wat @@ -20,96 +20,24 @@ ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 ;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail -;; sig1 = (i64 vmctx, i64, i32, i64) tail +;; sig1 = (i64 vmctx, i64) -> i64 tail ;; fn0 = colocated u805306368:24 sig0 -;; fn1 = colocated u805306368:2 sig1 +;; fn1 = colocated u805306368:25 sig1 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32): ;; @001f v5 = uextend.i64 v2 -;; v102 = iconst.i64 2 -;; v103 = ishl v5, v102 ; v102 = 2 -;; v100 = iconst.i64 32 -;; @001f v7 = ushr v103, v100 ; v100 = 32 +;; v110 = iconst.i64 2 +;; v111 = ishl v5, v110 ; v110 = 2 +;; v108 = iconst.i64 32 +;; @001f v7 = ushr v111, v108 ; v108 = 32 ;; @001f trapnz v7, user18 ;; @001f v4 = iconst.i32 20 -;; v109 = iconst.i32 2 -;; v110 = ishl v2, v109 ; v109 = 2 -;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20 +;; v117 = iconst.i32 2 +;; v118 = ishl v2, v117 ; v117 = 2 +;; @001f v9 = uadd_overflow_trap v4, v118, user18 ; v4 = 20 ;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 ;; @001f v12 = load.i32 notrap aligned v11 ;; @001f v13 = load.i32 notrap aligned v11+4 ;; @001f v19 = uextend.i64 v12 ;; @001f v14 = uextend.i64 v9 -;; @001f v15 = iconst.i64 15 -;; @001f v17 = iadd v14, v15 ; v15 = 15 -;; @001f v16 = iconst.i64 -16 -;; @001f v18 = band v17, v16 ; v16 = -16 -;; @001f v20 = iadd v19, v18 -;; @001f v21 = uextend.i64 v13 -;; @001f v22 = icmp ule v20, v21 -;; @001f brif v22, block2, block3 -;; -;; block2: -;; v118 = iconst.i32 15 -;; v119 = iadd.i32 v9, v118 ; v118 = 15 -;; v122 = iconst.i32 -16 -;; v123 = band v119, v122 ; v122 = -16 -;; v125 = iadd.i32 v12, v123 -;; @001f store notrap aligned region0 v125, v11 -;; v140 = iconst.i32 -1476395002 -;; v141 = load.i64 notrap aligned readonly can_move v0+8 -;; v142 = load.i64 notrap aligned readonly can_move v141+32 -;; @001f v36 = iadd v142, v19 -;; @001f store notrap aligned v140, v36 ; v140 = -1476395002 -;; v143 = load.i64 notrap aligned readonly can_move v0+40 -;; v144 = load.i32 notrap aligned readonly can_move v143 -;; @001f store notrap aligned v144, v36+4 -;; v145 = band.i64 v17, v16 ; v16 = -16 -;; @001f istore32 notrap aligned v145, v36+8 -;; @001f jump block4(v12, v36) -;; -;; block3 cold: -;; @001f v24 = iconst.i32 -1476395002 -;; @001f v26 = load.i64 notrap aligned readonly can_move v0+40 -;; @001f v27 = load.i32 notrap aligned readonly can_move v26 -;; @001f v28 = iconst.i32 16 -;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476395002, v28 = 16 -;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8 -;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32 -;; @001f v31 = uextend.i64 v29 -;; @001f v32 = iadd v30, v31 -;; @001f jump block4(v29, v32) -;; -;; block4(v41: i32, v42: i64): -;; v95 = stack_addr.i64 ss0 -;; store notrap v41, v95 -;; v94 = iconst.i64 16 -;; @001f v43 = iadd v42, v94 ; v94 = 16 -;; @001f store.i32 user2 v2, v43 -;; v77 = load.i32 notrap v95 -;; @001f trapz v77, user16 -;; v146 = load.i64 notrap aligned readonly can_move v0+8 -;; v147 = load.i64 notrap aligned readonly can_move v146+32 -;; @001f v46 = uextend.i64 v77 -;; @001f v48 = iadd v147, v46 -;; @001f v50 = iadd v48, v94 ; v94 = 16 -;; @001f v51 = load.i32 user2 readonly v50 -;; @001f v52 = uextend.i64 v51 -;; @001f v57 = icmp.i64 ugt v5, v52 -;; @001f trapnz v57, user17 -;; @001f v68 = load.i64 notrap aligned v146+40 -;; v85 = iconst.i64 20 -;; @001f v61 = iadd v48, v85 ; v85 = 20 -;; @001f v70 = uadd_overflow_trap v61, v103, user2 -;; @001f v69 = iadd v147, v68 -;; @001f v71 = icmp ugt v70, v69 -;; @001f trapnz v71, user2 -;; @001f v45 = iconst.i32 0 -;; @001f call fn1(v0, v61, v45, v103), stack_map=[i32 @ ss0+0] ; v45 = 0 -;; v74 = load.i32 notrap v95 -;; @0022 jump block1 -;; -;; block1: -;; @0022 return v74 -;; } diff --git a/tests/misc_testsuite/gc/array-fill-null-funcref.wast b/tests/misc_testsuite/gc/array-fill-null-funcref.wast new file mode 100644 index 000000000000..2e74c3384d16 --- /dev/null +++ b/tests/misc_testsuite/gc/array-fill-null-funcref.wast @@ -0,0 +1,10 @@ +;;! gc = true + +(module + (type $a (array funcref)) + (func (export "hi") (result funcref) + (array.get $a (array.new_default $a (i32.const 10)) (i32.const 3)) + ) +) + +(assert_return (invoke "hi") (ref.null func)) From edb8de096083d6499094a0956334d96db1e79248 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 26 May 2026 14:14:15 -0700 Subject: [PATCH 3/4] Move most module initialization to compiled code This commit is a large refactoring of how modules are initialized in Wasmtime. Notably all of the work done post-allocation, but pre-start, is now done in compiled code instead. This means that global initialization, active table segments, passive segment evaluation, etc, now all happens in compiled code. The primary motivation for this is to resolve some GC-related fuzz-bugs where initialization on the host is taking an excessively long time. A secondary motivation is to apply fuel metering and epoch yielding to these constructs in the same manner that normal wasm code has these applied. Much refactoring was needed in this commit to achieve this goal. Many primitives were transitioned from runtime state to exclusively compile-time state for example. Infrastructure was additionally added for a new kind of `FuncKey` corresponding to this one-off-use startup function. Overall though the net effect of this change is to mostly delete code since so much of the runtime is now no longer necessary. An example of this is that const-eval is now completely removed from the runtime as the fully-general const-evaluation now happens exclusively through compiled code. Special care was needed here for the static table and memory initialization that Wasmtime performs. For example there's a small dance between compile-time and run-time where at compile-time we don't know if static data segments should be applied, and it's only at run-time where we know if CoW is in effect. Additionally care was taken throughout this refactoring to avoid generating this new startup function unless it's necessary. It's hypothesized that skipping this function is going to be a worthwhile optimization, which means that one mode of startup is configured as "only necessary if memories say `needs_init()`". This is a bit tricky to document and it's a bit non-standard, but it should get the job done (and existing tests exercise this already). --- crates/cranelift/src/compiler.rs | 505 +++++++------ crates/cranelift/src/compiler/component.rs | 2 +- crates/cranelift/src/func_environ.rs | 688 +++++++++++++++--- crates/cranelift/src/func_environ/gc.rs | 14 + .../src/func_environ/gc/enabled/copying.rs | 13 + .../src/func_environ/gc/enabled/drc.rs | 206 +++--- .../src/func_environ/gc/enabled/null.rs | 12 + .../src/translate/func_translator.rs | 20 + crates/environ/src/compile/mod.rs | 35 +- .../environ/src/compile/module_artifacts.rs | 47 +- crates/environ/src/compile/module_environ.rs | 536 +++++++++----- crates/environ/src/compile/module_types.rs | 45 +- crates/environ/src/component/compiler.rs | 2 +- crates/environ/src/component/types_builder.rs | 26 +- crates/environ/src/key.rs | 35 +- crates/environ/src/module.rs | 269 +++---- crates/environ/src/module_artifacts.rs | 6 +- crates/environ/src/types.rs | 22 +- crates/environ/src/vmoffsets.rs | 89 ++- crates/wasmtime/src/compile.rs | 109 ++- crates/wasmtime/src/runtime/code.rs | 30 +- .../src/runtime/component/component.rs | 1 + .../src/runtime/gc/disabled/rooting.rs | 26 +- crates/wasmtime/src/runtime/instance.rs | 76 +- crates/wasmtime/src/runtime/instantiate.rs | 58 +- crates/wasmtime/src/runtime/module.rs | 9 +- crates/wasmtime/src/runtime/profiling.rs | 6 +- crates/wasmtime/src/runtime/trap.rs | 5 +- crates/wasmtime/src/runtime/values.rs | 11 - crates/wasmtime/src/runtime/vm.rs | 3 +- crates/wasmtime/src/runtime/vm/const_expr.rs | 499 ------------- crates/wasmtime/src/runtime/vm/cow.rs | 11 +- .../wasmtime/src/runtime/vm/gc/enabled/drc.rs | 3 +- crates/wasmtime/src/runtime/vm/instance.rs | 277 ++++--- .../src/runtime/vm/instance/allocator.rs | 450 +----------- crates/wasmtime/src/runtime/vm/libcalls.rs | 3 +- crates/wasmtime/src/runtime/vm/vmcontext.rs | 82 --- crates/winch/src/compiler.rs | 69 +- src/commands/objdump.rs | 1 + tests/all/arrays.rs | 1 + tests/all/cli_tests.rs | 2 +- tests/all/exceptions.rs | 1 + tests/all/exnrefs.rs | 1 + tests/all/fuel.wast | 70 +- tests/all/gc.rs | 4 + tests/all/memory.rs | 1 + tests/all/module.rs | 1 + tests/all/pooling_allocator.rs | 5 + tests/all/structs.rs | 1 + tests/all/table.rs | 1 + tests/disas/gc/array-init-data.wat | 35 +- tests/disas/gc/array-new-data.wat | 170 ++--- tests/disas/gc/array-new-default-anyref.wat | 110 +++ tests/disas/gc/array-new-default-exnref.wat | 110 +++ .../disas/gc/array-new-default-externref.wat | 110 +++ tests/disas/gc/array-new-default-funcref.wat | 87 +++ tests/disas/passive-data.wat | 35 +- tests/disas/startup-data-active.wat | 62 ++ tests/disas/startup-elem-active.wat | 77 ++ tests/disas/startup-global.wat | 42 ++ tests/disas/startup-passive-segment.wat | 50 ++ tests/disas/startup-start.wat | 42 ++ tests/disas/startup-table-initial-value.wat | 65 ++ .../disas/winch/x64/table/init_copy_drop.wat | 12 +- winch/codegen/src/codegen/mod.rs | 10 +- 65 files changed, 2965 insertions(+), 2441 deletions(-) delete mode 100644 crates/wasmtime/src/runtime/vm/const_expr.rs create mode 100644 tests/disas/startup-data-active.wat create mode 100644 tests/disas/startup-elem-active.wat create mode 100644 tests/disas/startup-global.wat create mode 100644 tests/disas/startup-passive-segment.wat create mode 100644 tests/disas/startup-start.wat create mode 100644 tests/disas/startup-table-initial-value.wat diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index a2c76c9f8a89..b3fd3941185e 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -39,9 +39,9 @@ use wasmtime_environ::{ Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody, DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder, FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, HostCall, Inlining, - InliningCompiler, ModulePC, ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapSection, - StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, WasmFuncType, - WasmValType, prelude::*, + InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize, + StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, + WasmFuncType, WasmValType, prelude::*, }; use wasmtime_unwinder::ExceptionTableBuilder; @@ -199,6 +199,230 @@ impl Compiler { builder.ins().call_indirect(sig, addr, args) } + + fn compile_wasm_to_array_trampoline( + &self, + wasm_func_ty: &WasmFuncType, + key: FuncKey, + symbol: &str, + ) -> Result { + log::trace!("compiling wasm-to-array trampoline: {key:?} = {symbol:?}"); + + let isa = &*self.isa; + let pointer_type = isa.pointer_type(); + let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables); + let array_call_sig = array_call_signature(isa); + + let mut compiler = self.function_compiler(); + let func = ir::Function::with_name_signature(key_to_name(key), wasm_call_sig); + let (mut builder, block0) = compiler.builder(func); + + let args = builder.func.dfg.block_params(block0).to_vec(); + let callee_vmctx = args[0]; + let caller_vmctx = args[1]; + + // We are exiting Wasm, so save our PC and FP. + // + // Assert that the caller vmctx really is a core Wasm vmctx, since + // that's what we are assuming with our offsets below. + self.debug_assert_vmctx_kind( + &mut builder, + caller_vmctx, + wasmtime_environ::VMCONTEXT_MAGIC, + ); + let ptr = isa.pointer_bytes(); + let vm_store_context = builder.ins().load( + pointer_type, + MemFlagsData::trusted(), + caller_vmctx, + i32::from(ptr.vmcontext_store_context()), + ); + save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr, vm_store_context); + + // Spill all wasm arguments to the stack in `ValRaw` slots. + let (args_base, args_len) = + self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]); + let args_len = builder.ins().iconst(pointer_type, i64::from(args_len)); + + // Load the actual callee out of the + // `VMArrayCallHostFuncContext::host_func`. + let ptr_size = isa.pointer_bytes(); + let callee = builder.ins().load( + pointer_type, + MemFlagsData::trusted(), + callee_vmctx, + ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(), + ); + + // Do an indirect call to the callee. + let callee_signature = builder.func.import_signature(array_call_sig); + let call = self.call_indirect_host( + &mut builder, + HostCall::ArrayCall, + callee_signature, + callee, + &[callee_vmctx, caller_vmctx, args_base, args_len], + ); + + // Increment the "execution version" on the VMStoreContext if + // guest debugging is enabled. + if self.tunables.debug_guest { + let vmstore_ctx_ptr = builder.ins().load( + pointer_type, + MemFlagsData::trusted().with_readonly(), + caller_vmctx, + i32::from(ptr_size.vmctx_store_context()), + ); + let old_version = builder.ins().load( + ir::types::I64, + MemFlagsData::trusted(), + vmstore_ctx_ptr, + i32::from(ptr_size.vmstore_context_execution_version()), + ); + let new_version = builder.ins().iadd_imm(old_version, 1); + builder.ins().store( + MemFlagsData::trusted(), + new_version, + vmstore_ctx_ptr, + i32::from(ptr_size.vmstore_context_execution_version()), + ); + } + + // Invoke `raise` if the callee (host) returned an error. + let succeeded = builder.func.dfg.inst_results(call)[0]; + self.raise_if_host_trapped(&mut builder, caller_vmctx, succeeded); + + // Return results from the array as native return values. + let results = + self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len); + builder.ins().return_(&results); + builder.finalize(); + + Ok(CompiledFunctionBody { + code: box_dyn_any_compiler_context(Some(compiler.cx)), + needs_gc_heap: false, + }) + } + + fn compile_wasm_to_builtin( + &self, + key: FuncKey, + symbol: &str, + ) -> Result { + log::trace!("compiling wasm-to-builtin trampoline: {key:?} = {symbol:?}"); + + let isa = &*self.isa; + let ptr_size = isa.pointer_bytes(); + let pointer_type = isa.pointer_type(); + let sigs = BuiltinFunctionSignatures::new(self); + + let (builtin_func_index, wasm_sig) = match key { + FuncKey::WasmToBuiltinTrampoline(builtin) => (builtin, sigs.wasm_signature(builtin)), + FuncKey::PatchableToBuiltinTrampoline(builtin) => { + let mut sig = sigs.wasm_signature(builtin); + // Patchable functions cannot return anything. We + // raise any errors that occur below so this is fine. + sig.returns.clear(); + sig.call_conv = CallConv::PreserveAll; + (builtin, sig) + } + _ => unreachable!(), + }; + let host_sig = sigs.host_signature(builtin_func_index); + + let mut compiler = self.function_compiler(); + let func = ir::Function::with_name_signature(key_to_name(key), wasm_sig.clone()); + let (mut builder, block0) = compiler.builder(func); + let vmctx = builder.block_params(block0)[0]; + + // Debug-assert that this is the right kind of vmctx, and then + // additionally perform the "routine of the exit trampoline" of saving + // fp/pc/etc. + self.debug_assert_vmctx_kind(&mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC); + let vm_store_context = builder.ins().load( + pointer_type, + MemFlagsData::trusted(), + vmctx, + ptr_size.vmcontext_store_context(), + ); + save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, vm_store_context); + + // Now it's time to delegate to the actual builtin. Forward all our own + // arguments to the libcall itself. + let args = builder.block_params(block0).to_vec(); + let call = self.call_builtin(&mut builder, vmctx, &args, builtin_func_index, host_sig); + let results = builder.func.dfg.inst_results(call).to_vec(); + + // Libcalls do not explicitly jump/raise on traps but instead return a + // code indicating whether they trapped or not. This means that it's the + // responsibility of the trampoline to check for an trapping return + // value and raise a trap as appropriate. With the `results` above check + // what `index` is and for each libcall that has a trapping return value + // process it here. + match builtin_func_index.trap_sentinel() { + Some(TrapSentinel::Falsy) => { + self.raise_if_host_trapped(&mut builder, vmctx, results[0]); + } + Some(TrapSentinel::NegativeTwo) => { + let ty = builder.func.dfg.value_type(results[0]); + let trapped = builder.ins().iconst(ty, -2); + let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped); + self.raise_if_host_trapped(&mut builder, vmctx, succeeded); + } + Some(TrapSentinel::Negative) => { + let ty = builder.func.dfg.value_type(results[0]); + let zero = builder.ins().iconst(ty, 0); + let succeeded = + builder + .ins() + .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero); + self.raise_if_host_trapped(&mut builder, vmctx, succeeded); + } + Some(TrapSentinel::NegativeOne) => { + let ty = builder.func.dfg.value_type(results[0]); + let minus_one = builder.ins().iconst(ty, -1); + let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one); + self.raise_if_host_trapped(&mut builder, vmctx, succeeded); + } + None => {} + } + + // And finally, return all the results of this libcall. + if !wasm_sig.returns.is_empty() { + builder.ins().return_(&results); + } else { + builder.ins().return_(&[]); + } + builder.finalize(); + + Ok(CompiledFunctionBody { + code: box_dyn_any_compiler_context(Some(compiler.cx)), + needs_gc_heap: false, + }) + } + + fn compile_module_startup( + &self, + translation: &ModuleTranslation<'_>, + types: &ModuleTypesBuilder, + key: FuncKey, + ty: &WasmFuncType, + ) -> Result { + let mut compiler = self.function_compiler(); + let context = &mut compiler.cx.codegen_context; + context.func.signature = wasm_call_signature(&*self.isa, ty, &self.tunables); + let (namespace, index) = key.into_raw_parts(); + context.func.name = UserFuncName::User(UserExternalName { namespace, index }); + let mut func_env = FuncEnvironment::new(self, translation, types, ty, key); + compiler + .cx + .func_translator + .translate_module_startup(&mut context.func, &mut func_env)?; + Ok(CompiledFunctionBody { + code: box_dyn_any_compiler_context(Some(compiler.cx)), + needs_gc_heap: func_env.needs_gc_heap(), + }) + } } fn box_dyn_any_compiled_function(f: CompiledFunction) -> Box { @@ -364,130 +588,82 @@ impl wasmtime_environ::Compiler for Compiler { }) } - fn compile_array_to_wasm_trampoline( + fn compile_trampoline( &self, - translation: &ModuleTranslation<'_>, - types: &ModuleTypesBuilder, - key: FuncKey, - symbol: &str, - ) -> Result { - let (module_index, def_func_index) = key.unwrap_array_to_wasm_trampoline(); - let func_index = translation.module.func_index(def_func_index); - let sig = translation.module.functions[func_index] - .signature - .unwrap_module_type_index(); - self.array_to_wasm_trampoline( - key, - FuncKey::DefinedWasmFunction(module_index, def_func_index), - types[sig].unwrap_func(), - symbol, - self.isa.pointer_bytes().vmctx_store_context().into(), - wasmtime_environ::VMCONTEXT_MAGIC, - ) - } - - fn compile_wasm_to_array_trampoline( - &self, - wasm_func_ty: &WasmFuncType, + translation: Option<&ModuleTranslation<'_>>, key: FuncKey, + types: &ModuleTypesBuilder, symbol: &str, ) -> Result { - log::trace!("compiling wasm-to-array trampoline: {key:?} = {symbol:?}"); - - let isa = &*self.isa; - let pointer_type = isa.pointer_type(); - let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables); - let array_call_sig = array_call_signature(isa); - - let mut compiler = self.function_compiler(); - let func = ir::Function::with_name_signature(key_to_name(key), wasm_call_sig); - let (mut builder, block0) = compiler.builder(func); + match key { + FuncKey::ArrayToWasmTrampoline(module_index, def_func_index) => { + let translation = translation.unwrap(); + let func_index = translation.module.func_index(def_func_index); + let sig = translation.module.functions[func_index] + .signature + .unwrap_module_type_index(); + self.array_to_wasm_trampoline( + key, + FuncKey::DefinedWasmFunction(module_index, def_func_index), + types[sig].unwrap_func(), + symbol, + self.isa.pointer_bytes().vmctx_store_context().into(), + wasmtime_environ::VMCONTEXT_MAGIC, + ) + } - let args = builder.func.dfg.block_params(block0).to_vec(); - let callee_vmctx = args[0]; - let caller_vmctx = args[1]; + FuncKey::WasmToArrayTrampoline(ty) => { + let ty = types[ty].unwrap_func(); + self.compile_wasm_to_array_trampoline(ty, key, symbol) + } + #[cfg(feature = "component-model")] + FuncKey::ResourceDropTrampoline => { + let ty = types.find_resource_drop_signature().unwrap(); + let ty = types[ty].unwrap_func(); + self.compile_wasm_to_array_trampoline(ty, key, symbol) + } - // We are exiting Wasm, so save our PC and FP. - // - // Assert that the caller vmctx really is a core Wasm vmctx, since - // that's what we are assuming with our offsets below. - self.debug_assert_vmctx_kind( - &mut builder, - caller_vmctx, - wasmtime_environ::VMCONTEXT_MAGIC, - ); - let ptr = isa.pointer_bytes(); - let vm_store_context = builder.ins().load( - pointer_type, - MemFlagsData::trusted(), - caller_vmctx, - i32::from(ptr.vmcontext_store_context()), - ); - save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr, vm_store_context); + FuncKey::DefinedWasmFunction(..) => { + panic!("use `compile_function` instead") + } - // Spill all wasm arguments to the stack in `ValRaw` slots. - let (args_base, args_len) = - self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]); - let args_len = builder.ins().iconst(pointer_type, i64::from(args_len)); + FuncKey::WasmToBuiltinTrampoline(..) | FuncKey::PatchableToBuiltinTrampoline(_) => { + self.compile_wasm_to_builtin(key, symbol) + } - // Load the actual callee out of the - // `VMArrayCallHostFuncContext::host_func`. - let ptr_size = isa.pointer_bytes(); - let callee = builder.ins().load( - pointer_type, - MemFlagsData::trusted(), - callee_vmctx, - ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(), - ); + FuncKey::PulleyHostCall(_) => unreachable!(), - // Do an indirect call to the callee. - let callee_signature = builder.func.import_signature(array_call_sig); - let call = self.call_indirect_host( - &mut builder, - HostCall::ArrayCall, - callee_signature, - callee, - &[callee_vmctx, caller_vmctx, args_base, args_len], - ); + #[cfg(feature = "component-model")] + FuncKey::ComponentTrampoline(..) | FuncKey::UnsafeIntrinsic(..) => { + unreachable!() + } - // Increment the "execution version" on the VMStoreContext if - // guest debugging is enabled. - if self.tunables.debug_guest { - let vmstore_ctx_ptr = builder.ins().load( - pointer_type, - MemFlagsData::trusted().with_readonly(), - caller_vmctx, - i32::from(ptr_size.vmctx_store_context()), - ); - let old_version = builder.ins().load( - ir::types::I64, - MemFlagsData::trusted(), - vmstore_ctx_ptr, - i32::from(ptr_size.vmstore_context_execution_version()), - ); - let new_version = builder.ins().iadd_imm(old_version, 1); - builder.ins().store( - MemFlagsData::trusted(), - new_version, - vmstore_ctx_ptr, - i32::from(ptr_size.vmstore_context_execution_version()), - ); + FuncKey::ModuleStartup(abi, module) => { + let translation = translation.unwrap(); + let ty = match translation.module.startup { + ModuleStartup::None => unreachable!(), + ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => { + t.unwrap_module_type_index() + } + }; + let ty = types[ty].unwrap_func(); + match abi { + // This is a think array-to-wasm shim around the actual + // implementation. + Abi::Array => self.array_to_wasm_trampoline( + key, + FuncKey::ModuleStartup(Abi::Wasm, module), + ty, + symbol, + self.isa.pointer_bytes().vmctx_store_context().into(), + wasmtime_environ::VMCONTEXT_MAGIC, + ), + // Delegate to a helper t o finish compiling this. + Abi::Wasm => self.compile_module_startup(translation, types, key, ty), + Abi::Patchable => unreachable!(), + } + } } - - // Invoke `raise` if the callee (host) returned an error. - let succeeded = builder.func.dfg.inst_results(call)[0]; - self.raise_if_host_trapped(&mut builder, caller_vmctx, succeeded); - - // Return results from the array as native return values. - let results = - self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len); - builder.ins().return_(&results); - builder.finalize(); - - Ok(CompiledFunctionBody { - code: box_dyn_any_compiler_context(Some(compiler.cx)), - needs_gc_heap: false, - }) } fn append_code( @@ -752,103 +928,6 @@ impl wasmtime_environ::Compiler for Compiler { self.isa.create_systemv_cie() } - fn compile_wasm_to_builtin( - &self, - key: FuncKey, - symbol: &str, - ) -> Result { - log::trace!("compiling wasm-to-builtin trampoline: {key:?} = {symbol:?}"); - - let isa = &*self.isa; - let ptr_size = isa.pointer_bytes(); - let pointer_type = isa.pointer_type(); - let sigs = BuiltinFunctionSignatures::new(self); - - let (builtin_func_index, wasm_sig) = match key { - FuncKey::WasmToBuiltinTrampoline(builtin) => (builtin, sigs.wasm_signature(builtin)), - FuncKey::PatchableToBuiltinTrampoline(builtin) => { - let mut sig = sigs.wasm_signature(builtin); - // Patchable functions cannot return anything. We - // raise any errors that occur below so this is fine. - sig.returns.clear(); - sig.call_conv = CallConv::PreserveAll; - (builtin, sig) - } - _ => unreachable!(), - }; - let host_sig = sigs.host_signature(builtin_func_index); - - let mut compiler = self.function_compiler(); - let func = ir::Function::with_name_signature(key_to_name(key), wasm_sig.clone()); - let (mut builder, block0) = compiler.builder(func); - let vmctx = builder.block_params(block0)[0]; - - // Debug-assert that this is the right kind of vmctx, and then - // additionally perform the "routine of the exit trampoline" of saving - // fp/pc/etc. - self.debug_assert_vmctx_kind(&mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC); - let vm_store_context = builder.ins().load( - pointer_type, - MemFlagsData::trusted(), - vmctx, - ptr_size.vmcontext_store_context(), - ); - save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, vm_store_context); - - // Now it's time to delegate to the actual builtin. Forward all our own - // arguments to the libcall itself. - let args = builder.block_params(block0).to_vec(); - let call = self.call_builtin(&mut builder, vmctx, &args, builtin_func_index, host_sig); - let results = builder.func.dfg.inst_results(call).to_vec(); - - // Libcalls do not explicitly jump/raise on traps but instead return a - // code indicating whether they trapped or not. This means that it's the - // responsibility of the trampoline to check for an trapping return - // value and raise a trap as appropriate. With the `results` above check - // what `index` is and for each libcall that has a trapping return value - // process it here. - match builtin_func_index.trap_sentinel() { - Some(TrapSentinel::Falsy) => { - self.raise_if_host_trapped(&mut builder, vmctx, results[0]); - } - Some(TrapSentinel::NegativeTwo) => { - let ty = builder.func.dfg.value_type(results[0]); - let trapped = builder.ins().iconst(ty, -2); - let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped); - self.raise_if_host_trapped(&mut builder, vmctx, succeeded); - } - Some(TrapSentinel::Negative) => { - let ty = builder.func.dfg.value_type(results[0]); - let zero = builder.ins().iconst(ty, 0); - let succeeded = - builder - .ins() - .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero); - self.raise_if_host_trapped(&mut builder, vmctx, succeeded); - } - Some(TrapSentinel::NegativeOne) => { - let ty = builder.func.dfg.value_type(results[0]); - let minus_one = builder.ins().iconst(ty, -1); - let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one); - self.raise_if_host_trapped(&mut builder, vmctx, succeeded); - } - None => {} - } - - // And finally, return all the results of this libcall. - if !wasm_sig.returns.is_empty() { - builder.ins().return_(&results); - } else { - builder.ins().return_(&[]); - } - builder.finalize(); - - Ok(CompiledFunctionBody { - code: box_dyn_any_compiler_context(Some(compiler.cx)), - needs_gc_heap: false, - }) - } - fn compiled_function_relocation_targets<'a>( &'a self, func: &'a dyn Any, diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index ec972bb7b340..d10c1370d396 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -1588,7 +1588,7 @@ impl TranslateTrap for TrapTranslator<'_> { } impl ComponentCompiler for Compiler { - fn compile_trampoline( + fn compile_component_trampoline( &self, component: &ComponentTranslation, types: &ComponentTypesBuilder, diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index e6e78366c49f..7f711037cac5 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -13,7 +13,7 @@ use crate::{ }; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; -use cranelift_codegen::ir::immediates::{Imm64, Offset32, V128Imm}; +use cranelift_codegen::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, V128Imm}; use cranelift_codegen::ir::{ self, BlockArg, Endianness, ExceptionTableData, ExceptionTableItem, types, }; @@ -29,10 +29,12 @@ use std::mem; use wasmparser::{FuncValidator, Operator, WasmFeatures, WasmModuleResources}; use wasmtime_core::math::f64_cvt_to_int_bounds; use wasmtime_environ::{ - BuiltinFunctionIndex, ComponentPC, DataIndex, DefinedFuncIndex, ElemIndex, - EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, - GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module, - ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex, + BuiltinFunctionIndex, ComponentPC, ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, + DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex, + FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex, + IndexType, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables, Module, + ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PassiveElemIndex, PtrSize, + RuntimeDataIndex, Table, TableIndex, TableInitialValue, TableSegment, TableSegmentElements, TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType, }; @@ -1481,9 +1483,13 @@ impl FuncEnvironment<'_> { if !self.module.globals[index].mutability { if let Some(index) = self.module.defined_global_index(index) { - let init = &self.module.global_initializers[index]; - if let Some(value) = init.const_eval() { - return GlobalVariable::Constant { value }; + if let Some((_, value)) = self + .module + .global_initializers + .iter() + .find(|(def_index, _)| *def_index == index) + { + return GlobalVariable::Constant { value: *value }; } } } @@ -2618,7 +2624,7 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let table_data = self.get_or_create_table(builder.func, table_index); let (dst, flags) = table_data.prepare_table_addr(self, builder, index); - self.emit_table_set(builder, table_index, dst, flags, value) + self.emit_table_set(builder, table_index, dst, flags, value, true) } /// Helper to store `value` into the table address at `addr` using `flags`. @@ -2632,19 +2638,32 @@ impl FuncEnvironment<'_> { addr: ir::Value, flags: ir::MemFlagsData, value: ir::Value, + initialized: bool, ) -> WasmResult<()> { let table = self.module.tables[table_index]; match table.ref_type.heap_type.top() { // GC-managed types. WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => { - gc::gc_compiler(self)?.translate_write_gc_reference( - self, - builder, - table.ref_type, - addr, - value, - flags, - ) + let mut gc = gc::gc_compiler(self)?; + if initialized { + gc.translate_write_gc_reference( + self, + builder, + table.ref_type, + addr, + value, + flags, + ) + } else { + gc.translate_init_gc_reference( + self, + builder, + table.ref_type, + addr, + value, + flags, + ) + } } // Function types. @@ -2693,7 +2712,16 @@ impl FuncEnvironment<'_> { val: ir::Value, len: ir::Value, ) -> WasmResult<()> { - self.translate_entity_fill(builder, table_index, dst, val, len) + self.translate_entity_fill( + builder, + CheckedEntity::Table { + table: table_index, + initialized: true, + }, + dst, + val, + len, + ) } pub fn translate_ref_i31( @@ -3164,6 +3192,21 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, global_index: GlobalIndex, val: ir::Value, + ) -> WasmResult<()> { + self.emit_global_set(builder, global_index, val, true) + } + + /// Helper to translate the `global.set` instruction. + /// + /// The main difference with `translate_global_set`, the main entrypoint, is + /// the `initialized` flag to indicate whether the global was previously + /// initialized to a valid value. + fn emit_global_set( + &mut self, + builder: &mut FunctionBuilder<'_>, + global_index: GlobalIndex, + val: ir::Value, + initialized: bool, ) -> WasmResult<()> { match self.get_or_create_global(builder.func, global_index) { GlobalVariable::Constant { .. } => { @@ -3197,14 +3240,26 @@ impl FuncEnvironment<'_> { let gv = builder.ins().global_value(self.pointer_type(), gv); let src = builder.ins().iadd_imm(gv, i64::from(offset)); - gc::gc_compiler(self)?.translate_write_gc_reference( - self, - builder, - ty, - src, - val, - ir::MemFlagsData::trusted(), - )? + let mut gc = gc::gc_compiler(self)?; + if initialized { + gc.translate_write_gc_reference( + self, + builder, + ty, + src, + val, + ir::MemFlagsData::trusted(), + )?; + } else { + gc.translate_init_gc_reference( + self, + builder, + ty, + src, + val, + ir::MemFlagsData::trusted(), + )?; + } } } Ok(()) @@ -3731,7 +3786,7 @@ impl FuncEnvironment<'_> { }, ); } - CheckedEntity::Table(table) => { + CheckedEntity::Table { table, initialized } => { self.emit_raw_array_or_table_fill( builder, entity, @@ -3739,7 +3794,14 @@ impl FuncEnvironment<'_> { val, len_ptr, &|env, builder, addr, value| { - env.emit_table_set(builder, table, addr, ir::MemFlagsData::trusted(), value) + env.emit_table_set( + builder, + table, + addr, + ir::MemFlagsData::trusted(), + value, + initialized, + ) }, )?; } @@ -3761,7 +3823,9 @@ impl FuncEnvironment<'_> { )?; } // Not allowed to be written to in wasm. - CheckedEntity::Data { .. } | CheckedEntity::Elem(_) => unreachable!(), + CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => { + unreachable!() + } } Ok(()) @@ -3993,7 +4057,7 @@ impl FuncEnvironment<'_> { // Lookup the passive data segment corresponding to this data segment. // If this is an active data segment then it already has length 0 so // there's nothing to do. - let passive_index = match self.translation.passive_data_map[seg_index] { + let runtime_index = match self.translation.runtime_data_map[seg_index] { Some(idx) => idx, None => return Ok(()), }; @@ -4006,7 +4070,7 @@ impl FuncEnvironment<'_> { ir::MemFlagsData::trusted(), new_length, vmctx, - i32::try_from(self.offsets.vmctx_passive_data_length(passive_index)).unwrap(), + i32::try_from(self.offsets.vmctx_runtime_data_length(runtime_index)).unwrap(), ); Ok(()) @@ -4080,7 +4144,9 @@ impl FuncEnvironment<'_> { CheckedEntity::Memory(_) => { assert!(matches!( src_entity, - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) )); // A memory's elements are bytes, so the element count is already // the byte length. @@ -4098,7 +4164,7 @@ impl FuncEnvironment<'_> { // Tables/arrays are sometimes a memcpy, sometimes a per-element // loop. Delegate further to figure that out. - CheckedEntity::Table(_) | CheckedEntity::Array { .. } => self + CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => self .emit_raw_array_or_table_copy( builder, dst_entity, @@ -4111,7 +4177,9 @@ impl FuncEnvironment<'_> { ), // Cannot copy into a data or element segment in wasm. - CheckedEntity::Data { .. } | CheckedEntity::Elem(_) => unreachable!(), + CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => { + unreachable!() + } } } @@ -4136,26 +4204,16 @@ impl FuncEnvironment<'_> { // Load the entity size, as `pointer_type`. let entity_size = match entity { CheckedEntity::Memory(i) => self.memory_size_in_bytes(&mut builder.cursor(), i), - CheckedEntity::Table(i) => { - let size = self.translate_table_size(builder.cursor(), i); + CheckedEntity::Table { table, .. } => { + let size = self.translate_table_size(builder.cursor(), table); self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), size, idx_type) } - CheckedEntity::Data { segment, .. } => match self.translation.passive_data_map[segment] + CheckedEntity::Data { segment, .. } => match self.translation.runtime_data_map[segment] { - Some(passive_index) => { - let vmctx = self.vmctx_val(&mut builder.cursor()); - let offset = - i32::try_from(self.offsets.vmctx_passive_data_length(passive_index)) - .unwrap(); - let flags = ir::MemFlagsData::trusted(); - match pointer_type { - I32 => builder.ins().load(I32, flags, vmctx, offset), - I64 => builder.ins().uload32(flags, vmctx, offset), - _ => unreachable!(), - } - } + Some(i) => self.load_runtime_data_length_as_pointer(builder, i), None => builder.ins().iconst(pointer_type, 0), }, + CheckedEntity::RuntimeData(i) => self.load_runtime_data_length_as_pointer(builder, i), CheckedEntity::Elem(i) => match self.translation.passive_elem_map[i] { Some(passive_index) => { let vmctx = self.vmctx_val(&mut builder.cursor()); @@ -4225,30 +4283,20 @@ impl FuncEnvironment<'_> { let heap = &self.heaps()[heap]; builder.ins().global_value(pointer_type, heap.base) } - CheckedEntity::Table(i) => { - let table = self.get_or_create_table(builder.func, i); + CheckedEntity::Table { table, .. } => { + let table = self.get_or_create_table(builder.func, table); builder.ins().global_value(pointer_type, table.base_gv) } - CheckedEntity::Data { segment, .. } => match self.translation.passive_data_map[segment] + CheckedEntity::Data { segment, .. } => match self.translation.runtime_data_map[segment] { - Some(passive_index) => { - let vmctx = self.vmctx_val(&mut builder.cursor()); - let offset = - i32::try_from(self.offsets.vmctx_passive_data_base(passive_index)).unwrap(); - builder.ins().load( - self.pointer_type(), - ir::MemFlagsData::trusted(), - vmctx, - offset, - ) - } - + Some(runtime_index) => self.load_runtime_data_base(builder, runtime_index), // Any address should do for an active data segment, but pick // something non-null for now. Note that the length of an active // data segment is always 0, so we know that the memcpy, if any, // will be 0 elements, so the actual value here doesn't matter. None => builder.ins().iconst(pointer_type, 1), }, + CheckedEntity::RuntimeData(i) => self.load_runtime_data_base(builder, i), // Element segments are quite similar to data segments just above. CheckedEntity::Elem(i) => match self.translation.passive_elem_map[i] { Some(passive_index) => { @@ -4295,6 +4343,41 @@ impl FuncEnvironment<'_> { Ok(builder.ins().iadd(base, byte_offset)) } + fn load_runtime_data_length( + &mut self, + builder: &mut FunctionBuilder<'_>, + runtime_index: RuntimeDataIndex, + ) -> ir::Value { + let vmctx = self.vmctx_val(&mut builder.cursor()); + let offset = i32::try_from(self.offsets.vmctx_runtime_data_length(runtime_index)).unwrap(); + let flags = ir::MemFlagsData::trusted(); + builder.ins().load(I32, flags, vmctx, offset) + } + + fn load_runtime_data_length_as_pointer( + &mut self, + builder: &mut FunctionBuilder<'_>, + runtime_index: RuntimeDataIndex, + ) -> ir::Value { + let length = self.load_runtime_data_length(builder, runtime_index); + self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), length, IndexType::I32) + } + + fn load_runtime_data_base( + &mut self, + builder: &mut FunctionBuilder<'_>, + runtime_index: RuntimeDataIndex, + ) -> ir::Value { + let vmctx = self.vmctx_val(&mut builder.cursor()); + let offset = i32::try_from(self.offsets.vmctx_runtime_data_base(runtime_index)).unwrap(); + builder.ins().load( + self.pointer_type(), + ir::MemFlagsData::trusted(), + vmctx, + offset, + ) + } + pub fn translate_table_copy( &mut self, builder: &mut FunctionBuilder<'_>, @@ -4304,7 +4387,20 @@ impl FuncEnvironment<'_> { src: ir::Value, len: ir::Value, ) -> WasmResult<()> { - self.translate_entity_copy(builder, dst_table_index, src_table_index, dst, src, len) + self.translate_entity_copy( + builder, + CheckedEntity::Table { + table: dst_table_index, + initialized: true, + }, + CheckedEntity::Table { + table: src_table_index, + initialized: true, + }, + dst, + src, + len, + ) } /// Emits a copy between two WebAssembly table or array entities. @@ -4377,7 +4473,7 @@ impl FuncEnvironment<'_> { // would mean that memcpy isn't suitable. If lazy init is // disabled though then funcrefs are just pointers so a // memcpy can be used. - CheckedEntity::Table(_) => self.tunables.table_lazy_init, + CheckedEntity::Table { .. } => self.tunables.table_lazy_init, // The GC heap has integers representing funcrefs, so memcpy // is fine. CheckedEntity::Array { .. } => false, @@ -4385,7 +4481,9 @@ impl FuncEnvironment<'_> { // can't work. CheckedEntity::Elem(_) => true, // Not possible - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => unreachable!(), + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) => unreachable!(), }, }, }; @@ -4439,7 +4537,10 @@ impl FuncEnvironment<'_> { // it's easier right now to share the internals of // `translate_table_get` which are a bit tricky with // funcrefs. - CheckedEntity::Table(i) => this.translate_table_get(builder, i, src_index)?, + CheckedEntity::Table { table, initialized } => { + assert!(initialized); + this.translate_table_get(builder, table, src_index)? + } CheckedEntity::Array { initialized, .. } => { assert!(initialized); let read_ty = src_entity.storage_type(this); @@ -4468,11 +4569,20 @@ impl FuncEnvironment<'_> { } } } - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => unreachable!(), + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) => unreachable!(), }; match dst_entity { - CheckedEntity::Table(i) => { - this.emit_table_set(builder, i, dst, ir::MemFlagsData::trusted(), val)?; + CheckedEntity::Table { table, initialized } => { + this.emit_table_set( + builder, + table, + dst, + ir::MemFlagsData::trusted(), + val, + initialized, + )?; } CheckedEntity::Array { initialized, .. } => { if initialized { @@ -4483,6 +4593,7 @@ impl FuncEnvironment<'_> { } CheckedEntity::Memory(_) | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) | CheckedEntity::Elem(_) => unreachable!(), } Ok(()) @@ -4842,7 +4953,10 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { self.translate_entity_copy( builder, - table_index, + CheckedEntity::Table { + table: table_index, + initialized: true, + }, CheckedEntity::Elem(ElemIndex::from_u32(seg_index)), dst, src, @@ -5645,6 +5759,386 @@ impl FuncEnvironment<'_> { pub fn is_reachable(&self) -> bool { self.stacks.reachable() } + + /// Generates the body of a `FuncKey::ModuleStartup(..)` function. + /// + /// This will perform all initialization of a module before any code in the + /// module itself starts executing. This will, for example, execute all + /// constant expressions for global initializers. + pub fn translate_module_startup(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> { + for (i, expr) in self.translation.global_initializers.iter() { + self.module_initialize_global(builder, *i, expr)?; + } + for (i, exprs) in self.translation.passive_elements.iter() { + self.module_initialize_passive_element(builder, i, exprs)?; + } + for (i, init) in self.translation.table_initialization.initial_values.iter() { + match init { + TableInitialValue::Null => {} + TableInitialValue::Expr(expr) => { + self.module_initialize_table_with_fill(builder, i, expr)?; + } + } + } + for segment in self.translation.table_initialization.segments.iter() { + self.module_initialize_table_with_segment(builder, segment)?; + } + match &self.translation.memory_init { + MemoryInit::Unprocessed(_) => unreachable!(), + MemoryInit::Processed(segments) => { + for (memory, offset, data) in segments.iter() { + self.module_initialize_memory_segment(builder, *memory, offset, *data)?; + } + } + } + if let Some(i) = self.translation.start_func { + self.module_start(builder, i)?; + } + Ok(()) + } + + /// Initializes all "complicated" globals in a module. + /// + /// This will translate the given constant expression and then initialize + /// the global. Note that this translation is done with the global not + /// initialized, meaning that GC barriers are slightly tweaked. + fn module_initialize_global( + &mut self, + builder: &mut FunctionBuilder, + global: DefinedGlobalIndex, + expr: &ConstExpr, + ) -> WasmResult<()> { + let index = self.module.global_index(global); + let val = self.translate_const_expr(builder, expr)?; + self.emit_global_set(builder, index, val, false) + } + + /// Initializes all passive element segments for a module. + /// + /// This will execute all constant expressions for all passive element + /// segments and initialize all `ValRaw` storage on the host. This + /// implements the semantics of how passive element segments are evaluated + /// once in a wasm instance. + fn module_initialize_passive_element( + &mut self, + builder: &mut FunctionBuilder, + elem: PassiveElemIndex, + exprs: &TableSegmentElements, + ) -> WasmResult<()> { + let vmctx = self.vmctx_val(&mut builder.cursor()); + let libcall = self + .builtin_functions + .passive_elem_segment_base(builder.func); + let idx = builder.ins().iconst(I32, i64::from(elem.as_u32())); + let call = builder.ins().call(libcall, &[vmctx, idx]); + let base = builder.func.dfg.first_result(call); + + // Values in `ValRaw` are always stored in little-endian. + let flags = ir::MemFlagsData::trusted().with_endianness(Endianness::Little); + + match exprs { + TableSegmentElements::Functions(indices) => { + for (i, func) in indices.iter().enumerate() { + let func = self.translate_ref_func(builder.cursor(), *func)?; + builder.ins().store( + flags, + func, + base, + i32::try_from(i.checked_mul(16).unwrap()).unwrap(), + ); + } + } + TableSegmentElements::Expressions { exprs, ty } => { + for (i, expr) in exprs.iter().enumerate() { + let val = self.translate_const_expr(builder, expr)?; + + let dst = builder + .ins() + .iadd_imm(base, i64::try_from(i.checked_mul(16).unwrap()).unwrap()); + match ty.heap_type.top() { + WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => { + let ty = WasmStorageType::Val(WasmValType::Ref(*ty)); + gc::init_field_at_addr(self, builder, ty, dst, val)?; + } + WasmHeapTopType::Func | WasmHeapTopType::Cont => { + builder.ins().store( + flags, + val, + base, + i32::try_from(i.checked_mul(16).unwrap()).unwrap(), + ); + } + } + } + } + } + Ok(()) + } + + /// Initializes all tables with non-null initializers. + /// + /// This function will morally execute a `table.fill` for the specified + /// defined table in this module for the result of the constant expression + /// provided. Note that the fill operation is done on an initialized table + /// which slightly alters the barriers emitted for GC types. + fn module_initialize_table_with_fill( + &mut self, + builder: &mut FunctionBuilder, + table: DefinedTableIndex, + init: &ConstExpr, + ) -> WasmResult<()> { + let table = self.module.table_index(table); + let ty = self.module.tables[table]; + let val = self.translate_const_expr(builder, init)?; + let dst = builder.ins().iconst(index_type_to_ir_type(ty.idx_type), 0); + let len = builder.ins().iconst( + index_type_to_ir_type(ty.idx_type), + ty.limits.min.cast_signed(), + ); + self.translate_entity_fill( + builder, + CheckedEntity::Table { + table, + initialized: false, + }, + dst, + val, + len, + ) + } + + /// Executes initialization for an active element segment in a module. + /// + /// Note that this isn't done for "precomputed" tables which won't have + /// initializers present. Additionally note that this doesn't use + /// `table.init` because the element segment won't actually be present + /// anywhere at runtime. + fn module_initialize_table_with_segment( + &mut self, + builder: &mut FunctionBuilder, + segment: &TableSegment, + ) -> WasmResult<()> { + let offset = self.translate_const_expr(builder, &segment.offset)?; + let segment_len = builder.ins().iconst( + index_type_to_ir_type(self.module.tables[segment.table_index].idx_type), + i64::try_from(segment.elements.len()).unwrap(), + ); + + // Check the bounds first before mutating anything. + self.translate_entity_bounds_check( + builder, + CheckedEntity::Table { + table: segment.table_index, + initialized: true, + }, + offset, + segment_len, + )?; + + // Re-use the `table.set` translation for making this a simple function + // to define. That re-executes the bounds check which is a bit + // inefficient, but can be optimized in the future. + match &segment.elements { + TableSegmentElements::Functions(indices) => { + for (i, func) in indices.iter().enumerate() { + let func = self.translate_ref_func(builder.cursor(), *func)?; + let index = builder.ins().iadd_imm(offset, i64::try_from(i).unwrap()); + self.translate_table_set(builder, segment.table_index, func, index)?; + } + } + TableSegmentElements::Expressions { exprs, ty: _ } => { + for (i, expr) in exprs.iter().enumerate() { + let val = self.translate_const_expr(builder, expr)?; + let index = builder.ins().iadd_imm(offset, i64::try_from(i).unwrap()); + self.translate_table_set(builder, segment.table_index, val, index)?; + } + } + } + Ok(()) + } + + /// Peform initialization of an active data segment in a module. + fn module_initialize_memory_segment( + &mut self, + builder: &mut FunctionBuilder, + memory: MemoryIndex, + offset: &MemorySegmentOffset, + data: RuntimeDataIndex, + ) -> WasmResult<()> { + let mut end = None; + let offset = match offset { + // This is a bit subtle, but the goal here is to make a dynamic + // deduction to see if this initialization is actually necessary + // based on state at runtime. With a static initializer the goal is + // to enable CoW initialization at runtime, but not all hosts have + // that configured or supported. If it's supported then we shouldn't + // do anything here, but if it's unsupported then this must actually + // execute initialization. + // + // The host already put the addresses of all data segments into the + // `VMContext` during `VMContext::initialize_vmctx`, and this reads + // out the base pointer. If it's null then the host doesn't actually + // need initialization, so this is skipped, and otherwise it's the + // actual base pointer that's going to be used. + MemorySegmentOffset::Static(n) => { + let current_block = builder.current_block().unwrap(); + let init_block = builder.create_block(); + let end_block = builder.create_block(); + end = Some(end_block); + + builder.ensure_inserted_block(); + builder.insert_block_after(init_block, current_block); + builder.insert_block_after(end_block, init_block); + + let base = self.load_runtime_data_base(builder, data); + let null = builder.ins().iconst(self.pointer_type(), 0); + let is_null = builder.ins().icmp(IntCC::Equal, base, null); + builder.ins().brif(is_null, end_block, &[], init_block, &[]); + builder.switch_to_block(init_block); + builder.seal_block(init_block); + let ty = index_type_to_ir_type(self.module.memories[memory].idx_type); + builder.ins().iconst(ty, n.cast_signed()) + } + MemorySegmentOffset::Expr(expr) => self.translate_const_expr(builder, expr)?, + }; + + // Model the initialization here as a `memory.init`. + let len = self.load_runtime_data_length(builder, data); + let start = builder.ins().iconst(I32, 0); + self.translate_entity_copy(builder, memory, data, offset, start, len)?; + + // Finalize control-flow for the `MemorySegmentOffset::Static` case + // above. + if let Some(end) = end { + builder.ins().jump(end, &[]); + builder.switch_to_block(end); + builder.seal_block(end); + } + Ok(()) + } + + /// Translates the final step of module initialization, executing the + /// `start` function. + fn module_start(&mut self, builder: &mut FunctionBuilder, func: FuncIndex) -> WasmResult<()> { + // Manuall manage fuel around the call as the `Call` opcode does for + // normal wasm to ensure that it's correctly accounted for. + if self.tunables.consume_fuel { + self.fuel_consumed += 1; + self.fuel_increment_var(builder); + self.fuel_save_from_var(builder); + } + let ty = self.module.functions[func] + .signature + .unwrap_module_type_index(); + let sig_ref = self.get_or_create_interned_sig_ref(builder.func, ty); + self.translate_call(builder, Default::default(), func, sig_ref, &[])?; + if self.tunables.consume_fuel { + self.fuel_load_into_var(builder); + } + Ok(()) + } + + /// Helper function to evaluate `expr` into a value. + /// + /// This primarily reuses all the `translate_*` functions elsewhere on this + /// builder toe deduplicate implementation details. + fn translate_const_expr( + &mut self, + builder: &mut FunctionBuilder, + expr: &ConstExpr, + ) -> WasmResult { + let mut stack = Vec::new(); + for op in expr.ops() { + if self.tunables.consume_fuel { + self.fuel_consumed += 1; + } + match op { + ConstOp::I32Const(i) => { + stack.push(builder.ins().iconst(I32, i64::from(*i))); + } + ConstOp::I64Const(i) => { + stack.push(builder.ins().iconst(I64, *i)); + } + ConstOp::F32Const(i) => { + stack.push(builder.ins().f32const(Ieee32::with_bits(*i))); + } + ConstOp::F64Const(i) => { + stack.push(builder.ins().f64const(Ieee64::with_bits(*i))); + } + ConstOp::V128Const(i) => { + let data = i.to_le_bytes().to_vec().into(); + let handle = builder.func.dfg.constants.insert(data); + stack.push(builder.ins().vconst(ir::types::I8X16, handle)); + } + ConstOp::GlobalGet(i) => { + stack.push(self.translate_global_get(builder, *i)?); + } + ConstOp::RefI31 => { + let val = stack.pop().unwrap(); + stack.push(self.translate_ref_i31(builder.cursor(), val)?); + } + ConstOp::RefNull(ty) => { + stack.push(self.translate_ref_null(builder.cursor(), *ty)?); + } + ConstOp::RefFunc(i) => { + stack.push(self.translate_ref_func(builder.cursor(), *i)?); + } + ConstOp::I32Add | ConstOp::I64Add => { + let b = stack.pop().unwrap(); + let a = stack.pop().unwrap(); + stack.push(builder.ins().iadd(a, b)); + } + ConstOp::I32Sub | ConstOp::I64Sub => { + let b = stack.pop().unwrap(); + let a = stack.pop().unwrap(); + stack.push(builder.ins().isub(a, b)); + } + ConstOp::I32Mul | ConstOp::I64Mul => { + let b = stack.pop().unwrap(); + let a = stack.pop().unwrap(); + stack.push(builder.ins().imul(a, b)); + } + ConstOp::StructNew { struct_type_index } => { + let arity = self.struct_fields_len(*struct_type_index)?; + let fields = stack.drain(stack.len() - arity..).collect::<_>(); + stack.push(self.translate_struct_new(builder, *struct_type_index, fields)?); + } + ConstOp::StructNewDefault { struct_type_index } => { + stack.push(self.translate_struct_new_default(builder, *struct_type_index)?); + } + ConstOp::ArrayNew { array_type_index } => { + let len = stack.pop().unwrap(); + let elem = stack.pop().unwrap(); + stack.push(self.translate_array_new(builder, *array_type_index, elem, len)?); + } + ConstOp::ArrayNewDefault { array_type_index } => { + let len = stack.pop().unwrap(); + stack.push(self.translate_array_new_default( + builder, + *array_type_index, + len, + )?); + } + ConstOp::ArrayNewFixed { + array_type_index, + array_size, + } => { + let array_size = usize::try_from(*array_size).unwrap(); + let elems = stack.drain(stack.len() - array_size..).collect::>(); + stack.push(self.translate_array_new_fixed( + builder, + *array_type_index, + &elems, + )?); + } + ConstOp::ExternConvertAny => {} + ConstOp::AnyConvertExtern => {} + } + } + let ret = stack.pop().unwrap(); + assert!(stack.is_empty()); + Ok(ret) + } } // Helper function to convert an `IndexType` to an `ir::Type`. @@ -5698,7 +6192,12 @@ enum CheckedEntity { /// A WebAssembly linear memory. Memory(MemoryIndex), /// A WebAssembly table. - Table(TableIndex), + Table { + /// The index of the table that's being accessed. + table: TableIndex, + /// Whether or not this table has been initialized yet. + initialized: bool, + }, /// A WebAssembly data segment loaded in chunks of `element_size` bytes. /// /// For `memory.init` this will have `element_size = 1`, but for @@ -5711,6 +6210,11 @@ enum CheckedEntity { /// or possibly more for array initialization. element_size: u32, }, + /// A refinement of `Data` above which uses a `RuntimeDataIndex`. + /// + /// This isn't reachable from core wasm but is used during module + /// startup. + RuntimeData(RuntimeDataIndex), /// A WebAssembly passive element segment. Elem(ElemIndex), /// An `arrayref` with the specified type. @@ -5731,9 +6235,9 @@ impl From for CheckedEntity { } } -impl From for CheckedEntity { - fn from(table_index: TableIndex) -> Self { - CheckedEntity::Table(table_index) +impl From for CheckedEntity { + fn from(runtime_data_index: RuntimeDataIndex) -> Self { + CheckedEntity::RuntimeData(runtime_data_index) } } @@ -5742,10 +6246,11 @@ impl CheckedEntity { fn index_type(&self, env: &FuncEnvironment) -> IndexType { match *self { CheckedEntity::Memory(i) => env.memory(i).idx_type, - CheckedEntity::Table(i) => env.table(i).idx_type, - CheckedEntity::Data { .. } | CheckedEntity::Array { .. } | CheckedEntity::Elem(_) => { - IndexType::I32 - } + CheckedEntity::Table { table, .. } => env.table(table).idx_type, + CheckedEntity::Data { .. } + | CheckedEntity::Array { .. } + | CheckedEntity::Elem(_) + | CheckedEntity::RuntimeData(_) => IndexType::I32, } } @@ -5756,9 +6261,9 @@ impl CheckedEntity { func: &mut ir::Function, ) -> WasmResult { Ok(match *self { - CheckedEntity::Memory(_) => 1, + CheckedEntity::Memory(_) | CheckedEntity::RuntimeData(_) => 1, CheckedEntity::Data { element_size, .. } => element_size, - CheckedEntity::Table(table) => env.get_or_create_table(func, table).element_size, + CheckedEntity::Table { table, .. } => env.get_or_create_table(func, table).element_size, CheckedEntity::Array { ty, .. } => env.array_layout(ty)?.elem_size, CheckedEntity::Elem(_) => 16, }) @@ -5770,8 +6275,9 @@ impl CheckedEntity { CheckedEntity::Memory(_) | CheckedEntity::Data { element_size: 1, .. - } => WasmStorageType::I8, - CheckedEntity::Table(table) => { + } + | CheckedEntity::RuntimeData(_) => WasmStorageType::I8, + CheckedEntity::Table { table, .. } => { WasmStorageType::Val(WasmValType::Ref(env.table(table).ref_type)) } CheckedEntity::Array { ty, .. } => { @@ -5786,10 +6292,10 @@ impl CheckedEntity { /// Returns the trap code to use when an index into this entity is out-of-bounds. fn oob_trap_code(&self) -> ir::TrapCode { match self { - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => { - ir::TrapCode::HEAP_OUT_OF_BOUNDS - } - CheckedEntity::Table(_) | CheckedEntity::Elem(_) => TRAP_TABLE_OUT_OF_BOUNDS, + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) => ir::TrapCode::HEAP_OUT_OF_BOUNDS, + CheckedEntity::Table { .. } | CheckedEntity::Elem(_) => TRAP_TABLE_OUT_OF_BOUNDS, CheckedEntity::Array { .. } => TRAP_ARRAY_OUT_OF_BOUNDS, } } @@ -5798,7 +6304,9 @@ impl CheckedEntity { /// bit patterns. fn allows_memset(&self, env: &FuncEnvironment) -> bool { match self { - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => true, + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) => true, CheckedEntity::Array { ty, .. } => { let array_ty = env.types.unwrap_array(*ty).unwrap(); // Most GC references need barriers, funcrefs need intern-ing, @@ -5811,7 +6319,7 @@ impl CheckedEntity { CheckedEntity::Elem(_) => false, // Tables that are lazily initialized can't be memset because the // initialized bit needs to be set when storing values. - CheckedEntity::Table(_) => !env.tunables.table_lazy_init, + CheckedEntity::Table { .. } => !env.tunables.table_lazy_init, } } } diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index 9857146f0361..7dd7b5905f42 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -162,6 +162,20 @@ pub trait GcCompiler { flags: ir::MemFlagsData, ) -> WasmResult<()>; + /// Same as [`Self::translate_write_gc_reference`] except that the + /// destination address has not been previously initialized. + /// + /// This will, for example, skip barriers on the destination address. + fn translate_init_gc_reference( + &mut self, + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ty: WasmRefType, + dst: ir::Value, + new_val: ir::Value, + flags: ir::MemFlagsData, + ) -> WasmResult<()>; + /// Stores `val` into `addr` with the `ty` specified. /// /// This initializes a previously-uninitialized field, so barriers on the diff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/enabled/copying.rs index 5c0e89a2445a..79cbe1841e49 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/copying.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/copying.rs @@ -400,6 +400,19 @@ impl GcCompiler for CopyingCompiler { unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) } + fn translate_init_gc_reference( + &mut self, + _func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ty: WasmRefType, + dst: ir::Value, + new_val: ir::Value, + flags: ir::MemFlagsData, + ) -> WasmResult<()> { + // No write barrier needed for the copying collector. + unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) + } + fn init_field( &mut self, func_env: &mut FuncEnvironment<'_>, diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/enabled/drc.rs index 7938257a4b68..d59c3dc6b2dc 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/drc.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/drc.rs @@ -330,109 +330,6 @@ impl DrcCompiler { ); builder.ins().store(GC_MEMFLAGS, new_reserved, ptr, 0); } - - /// Write to an uninitialized GC reference field, initializing it. - /// - /// ```text - /// *dst = new_val - /// ``` - /// - /// Doesn't need to do a full write barrier: we don't have an old reference - /// that is being overwritten and needs its refcount decremented, just a new - /// reference whose count should be incremented. - fn translate_init_gc_reference( - &mut self, - func_env: &mut FuncEnvironment<'_>, - builder: &mut FunctionBuilder, - ty: WasmRefType, - dst: ir::Value, - new_val: ir::Value, - flags: ir::MemFlagsData, - ) -> WasmResult<()> { - let (ref_ty, _) = func_env.reference_type(ty.heap_type); - - // Special case for references to uninhabited bottom types: see - // `translate_write_gc_reference` for details. - if let WasmHeapType::None = ty.heap_type { - if ty.nullable { - let null = builder.ins().iconst(ref_ty, 0); - builder.ins().store(flags, null, dst, 0); - } else { - let zero = builder.ins().iconst(ir::types::I32, 0); - builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT); - } - return Ok(()); - }; - - // Special case for `i31ref`s: no need for any barriers. - if let WasmHeapType::I31 = ty.heap_type { - return unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags); - } - - // Our initialization barrier for GC references being copied out of the - // stack and initializing a table/global/struct field/etc... is roughly - // equivalent to the following pseudo-CLIF: - // - // ``` - // current_block: - // ... - // let new_val_is_null_or_i31 = ... - // brif new_val_is_null_or_i31, continue_block, inc_ref_block - // - // inc_ref_block: - // let ref_count = load new_val.ref_count - // let new_ref_count = iadd_imm ref_count, 1 - // store new_val.ref_count, new_ref_count - // jump check_old_val_block - // - // continue_block: - // store dst, new_val - // ... - // ``` - // - // This write barrier is responsible for ensuring that the new value's - // ref count is incremented now that the table/global/struct/etc... is - // holding onto it. - - let current_block = builder.current_block().unwrap(); - let inc_ref_block = builder.create_block(); - let continue_block = builder.create_block(); - - builder.ensure_inserted_block(); - builder.insert_block_after(inc_ref_block, current_block); - builder.insert_block_after(continue_block, inc_ref_block); - - // Current block: check whether the new value is non-null and - // non-i31. If so, branch to the `inc_ref_block`. - log::trace!("DRC initialization barrier: check if the value is null or i31"); - let new_val_is_null_or_i31 = func_env.gc_ref_is_null_or_i31(builder, ty, new_val); - builder.ins().brif( - new_val_is_null_or_i31, - continue_block, - &[], - inc_ref_block, - &[], - ); - - // Block to increment the ref count of the new value when it is non-null - // and non-i31. - builder.switch_to_block(inc_ref_block); - builder.seal_block(inc_ref_block); - log::trace!("DRC initialization barrier: increment the ref count of the initial value"); - self.mutate_ref_count(func_env, builder, new_val, 1); - builder.ins().jump(continue_block, &[]); - - // Join point after we're done with the GC barrier: do the actual store - // to initialize the field. - builder.switch_to_block(continue_block); - builder.seal_block(continue_block); - log::trace!( - "DRC initialization barrier: finally, store into {dst:?} to initialize the field" - ); - unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)?; - - Ok(()) - } } impl GcCompiler for DrcCompiler { @@ -959,6 +856,109 @@ impl GcCompiler for DrcCompiler { Ok(()) } + /// Write to an uninitialized GC reference field, initializing it. + /// + /// ```text + /// *dst = new_val + /// ``` + /// + /// Doesn't need to do a full write barrier: we don't have an old reference + /// that is being overwritten and needs its refcount decremented, just a new + /// reference whose count should be incremented. + fn translate_init_gc_reference( + &mut self, + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ty: WasmRefType, + dst: ir::Value, + new_val: ir::Value, + flags: ir::MemFlagsData, + ) -> WasmResult<()> { + let (ref_ty, _) = func_env.reference_type(ty.heap_type); + + // Special case for references to uninhabited bottom types: see + // `translate_write_gc_reference` for details. + if let WasmHeapType::None = ty.heap_type { + if ty.nullable { + let null = builder.ins().iconst(ref_ty, 0); + builder.ins().store(flags, null, dst, 0); + } else { + let zero = builder.ins().iconst(ir::types::I32, 0); + builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT); + } + return Ok(()); + }; + + // Special case for `i31ref`s: no need for any barriers. + if let WasmHeapType::I31 = ty.heap_type { + return unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags); + } + + // Our initialization barrier for GC references being copied out of the + // stack and initializing a table/global/struct field/etc... is roughly + // equivalent to the following pseudo-CLIF: + // + // ``` + // current_block: + // ... + // let new_val_is_null_or_i31 = ... + // brif new_val_is_null_or_i31, continue_block, inc_ref_block + // + // inc_ref_block: + // let ref_count = load new_val.ref_count + // let new_ref_count = iadd_imm ref_count, 1 + // store new_val.ref_count, new_ref_count + // jump check_old_val_block + // + // continue_block: + // store dst, new_val + // ... + // ``` + // + // This write barrier is responsible for ensuring that the new value's + // ref count is incremented now that the table/global/struct/etc... is + // holding onto it. + + let current_block = builder.current_block().unwrap(); + let inc_ref_block = builder.create_block(); + let continue_block = builder.create_block(); + + builder.ensure_inserted_block(); + builder.insert_block_after(inc_ref_block, current_block); + builder.insert_block_after(continue_block, inc_ref_block); + + // Current block: check whether the new value is non-null and + // non-i31. If so, branch to the `inc_ref_block`. + log::trace!("DRC initialization barrier: check if the value is null or i31"); + let new_val_is_null_or_i31 = func_env.gc_ref_is_null_or_i31(builder, ty, new_val); + builder.ins().brif( + new_val_is_null_or_i31, + continue_block, + &[], + inc_ref_block, + &[], + ); + + // Block to increment the ref count of the new value when it is non-null + // and non-i31. + builder.switch_to_block(inc_ref_block); + builder.seal_block(inc_ref_block); + log::trace!("DRC initialization barrier: increment the ref count of the initial value"); + self.mutate_ref_count(func_env, builder, new_val, 1); + builder.ins().jump(continue_block, &[]); + + // Join point after we're done with the GC barrier: do the actual store + // to initialize the field. + builder.switch_to_block(continue_block); + builder.seal_block(continue_block); + log::trace!( + "DRC initialization barrier: finally, store into {dst:?} to initialize the field" + ); + unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)?; + + Ok(()) + } + /// Write to an uninitialized field or element inside a GC object. fn init_field( &mut self, diff --git a/crates/cranelift/src/func_environ/gc/enabled/null.rs b/crates/cranelift/src/func_environ/gc/enabled/null.rs index 70818ab69abe..789bdfb8eac5 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/null.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/null.rs @@ -371,6 +371,18 @@ impl GcCompiler for NullCompiler { unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) } + fn translate_init_gc_reference( + &mut self, + _func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ty: WasmRefType, + dst: ir::Value, + new_val: ir::Value, + flags: ir::MemFlagsData, + ) -> WasmResult<()> { + unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) + } + fn init_field( &mut self, func_env: &mut FuncEnvironment<'_>, diff --git a/crates/cranelift/src/translate/func_translator.rs b/crates/cranelift/src/translate/func_translator.rs index b6875745504f..80323def5c14 100644 --- a/crates/cranelift/src/translate/func_translator.rs +++ b/crates/cranelift/src/translate/func_translator.rs @@ -96,6 +96,26 @@ impl FuncTranslator { log::trace!("translated Wasm to CLIF:\n{}", func.display()); Ok(()) } + + /// Translate the `FuncKey::ModuleStartup` function. + pub fn translate_module_startup( + &mut self, + func: &mut ir::Function, + environ: &mut FuncEnvironment<'_>, + ) -> WasmResult<()> { + let mut builder = FunctionBuilder::new(func, &mut self.func_ctx); + let entry_block = builder.create_block(); + builder.append_block_params_for_function_params(entry_block); + builder.switch_to_block(entry_block); + builder.seal_block(entry_block); + builder.ensure_inserted_block(); + environ.before_translate_function(&mut builder)?; + environ.translate_module_startup(&mut builder)?; + environ.after_translate_function(&mut builder)?; + builder.ins().return_(&[]); + builder.finalize(); + Ok(()) + } } /// Declare local variables for the signature parameters that correspond to WebAssembly locals. diff --git a/crates/environ/src/compile/mod.rs b/crates/environ/src/compile/mod.rs index d3cd69a87234..4dc2fa6d01c3 100644 --- a/crates/environ/src/compile/mod.rs +++ b/crates/environ/src/compile/mod.rs @@ -5,7 +5,7 @@ use crate::error::Result; use crate::prelude::*; use crate::{ DefinedFuncIndex, FlagValue, FuncKey, FunctionLoc, ObjectKind, PrimaryMap, StaticModuleIndex, - TripleExt, Tunables, WasmError, WasmFuncType, obj, + TripleExt, Tunables, WasmError, obj, }; use object::write::{Object, SymbolId}; use object::{Architecture, BinaryFormat, FileFlags}; @@ -261,38 +261,11 @@ pub trait Compiler: Send + Sync { /// /// The trampoline should save the necessary state to record the /// host-to-Wasm transition (e.g. registers used for fast stack walking). - fn compile_array_to_wasm_trampoline( - &self, - translation: &ModuleTranslation<'_>, - types: &ModuleTypesBuilder, - key: FuncKey, - symbol: &str, - ) -> Result; - - /// Compile a trampoline for a Wasm caller calling a array callee with the - /// given signature. - /// - /// The trampoline should save the necessary state to record the - /// Wasm-to-host transition (e.g. registers used for fast stack walking). - fn compile_wasm_to_array_trampoline( - &self, - wasm_func_ty: &WasmFuncType, - key: FuncKey, - symbol: &str, - ) -> Result; - - /// Creates a trampoline that can be used to call Wasmtime's implementation - /// of the builtin function specified by `index`. - /// - /// The trampoline created can technically have any ABI but currently has - /// the native ABI. This will then perform all the necessary duties of an - /// exit trampoline from wasm and then perform the actual dispatch to the - /// builtin function. Builtin functions in Wasmtime are stored in an array - /// in all `VMContext` pointers, so the call to the host is an indirect - /// call. - fn compile_wasm_to_builtin( + fn compile_trampoline( &self, + translation: Option<&ModuleTranslation<'_>>, key: FuncKey, + types: &ModuleTypesBuilder, symbol: &str, ) -> Result; diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rs index ca409dae9791..c9e1f302adfc 100644 --- a/crates/environ/src/compile/module_artifacts.rs +++ b/crates/environ/src/compile/module_artifacts.rs @@ -5,8 +5,7 @@ use crate::WasmChecksum; use crate::error::{Result, bail}; use crate::prelude::*; use crate::{ - CompiledModuleInfo, DebugInfoData, FunctionName, MemoryInitialization, Metadata, - ModuleTranslation, Tunables, obj, + CompiledModuleInfo, DebugInfoData, FunctionName, Metadata, ModuleTranslation, Tunables, obj, }; use object::SectionKind; use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; @@ -118,9 +117,8 @@ impl<'a> ObjectBuilder<'a> { mut module, debuginfo, has_unparsed_debuginfo, - data, data_align, - passive_data, + runtime_data, wasm, .. } = translation; @@ -128,20 +126,15 @@ impl<'a> ObjectBuilder<'a> { // Place all data from the wasm module into a section which will the // source of the data later at runtime. This additionally keeps track of // the offset of - let mut total_data_len = 0; let data_offset = self .obj .append_section_data(self.data, &[], data_align.unwrap_or(1)); - for (i, data) in data.iter().enumerate() { - // The first data segment has its alignment specified as the alignment - // for the entire section, but everything afterwards is adjacent so it - // has alignment of 1. + for (i, (_, data)) in runtime_data.iter().enumerate() { + // The first data segment has its alignment specified as the + // alignment for the entire section, but everything afterwards is + // adjacent so it has alignment of 1. let align = if i == 0 { data_align.unwrap_or(1) } else { 1 }; self.obj.append_section_data(self.data, data, align); - total_data_len += data.len(); - } - for data in passive_data.iter() { - self.obj.append_section_data(self.data, data, 1); } // If any names are present in the module then the `ELF_NAME_DATA` section @@ -172,34 +165,12 @@ impl<'a> ObjectBuilder<'a> { } } - // Data offsets in `MemoryInitialization` are offsets within the - // `translation.data` list concatenated which is now present in the data - // segment that's appended to the object. Increase the offsets by - // `self.data_size` to account for any previously added module. - let data_offset = u32::try_from(data_offset).unwrap(); - match &mut module.memory_initialization { - MemoryInitialization::Segmented(list) => { - for segment in list { - segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); - segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); - } - } - MemoryInitialization::Static { map } => { - for (_, segment) in map { - if let Some(segment) = segment { - segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); - segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); - } - } - } - } - // Data offsets for passive data are relative to the start of - // `translation.passive_data` which was appended to the data segment + // `translation.runtime_data` which was appended to the data segment // of this object, after active data in `translation.data`. Update the // offsets to account prior modules added in addition to active data. - let data_offset = data_offset + u32::try_from(total_data_len).unwrap(); - for (_, range) in module.passive_data.iter_mut() { + let data_offset = u32::try_from(data_offset).unwrap(); + for (_, range) in module.runtime_data.iter_mut() { range.start = range.start.checked_add(data_offset).unwrap(); range.end = range.end.checked_add(data_offset).unwrap(); } diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index be2fc09c0d8b..b5b02b9e6feb 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -1,20 +1,19 @@ use crate::error::{OutOfMemory, Result, bail}; use crate::module::{ - FuncRefIndex, Initializer, MemoryInitialization, MemoryInitializer, Module, TableSegment, - TableSegmentElements, + FuncRefIndex, Initializer, MemoryInitialization, Module, TableSegment, TableSegmentElements, }; use crate::prelude::*; use crate::{ - ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, ElemIndex, EngineOrModuleTypeIndex, - EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType, InitMemory, MemoryIndex, - ModuleInternedTypeIndex, ModuleTypesBuilder, PanicOnOom as _, PassiveDataIndex, - PassiveElemIndex, PrimaryMap, SizeOverflow, StaticMemoryInitializer, StaticModuleIndex, - TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, WasmError, + ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, DefinedGlobalIndex, ElemIndex, + EngineOrModuleTypeIndex, EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType, + MemoryIndex, MemoryInitializer, ModuleInternedTypeIndex, ModuleStartup, ModuleTypesBuilder, + PanicOnOom as _, PassiveElemIndex, PrimaryMap, RuntimeDataIndex, StaticModuleIndex, TableIndex, + TableInitialValue, TableInitialization, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, WasmHeapTopType, WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter, }; +use alloc::borrow::Cow; use cranelift_entity::SecondaryMap; use cranelift_entity::packed_option::ReservedValue; -use std::borrow::Cow; use std::collections::HashMap; use std::mem; use std::path::PathBuf; @@ -84,13 +83,6 @@ pub struct ModuleTranslation<'data> { /// configuration. pub has_unparsed_debuginfo: bool, - /// List of data segments found in this module which should be concatenated - /// together for the final compiled artifact. - /// - /// These data segments, when concatenated, are indexed by the - /// `MemoryInitializer` type. - pub data: Vec>, - /// The desired alignment of `data` in the final data section of the object /// file that we'll emit. /// @@ -100,20 +92,21 @@ pub struct ModuleTranslation<'data> { pub data_align: Option, /// Map from a data segment to whether it's a passive data segment or not. - pub passive_data_map: SecondaryMap>, + pub runtime_data_map: SecondaryMap>, /// Map from an elem segment to whether it's a passive elem segment or not. pub passive_elem_map: SecondaryMap>, - /// Total size of all data pushed onto `data` so far. - total_data: u32, - /// List of passive element segments found in this module which will get /// concatenated for the final artifact. - pub passive_data: Vec<&'data [u8]>, + pub runtime_data: PrimaryMap>, - /// Total size of all passive data pushed into `passive_data` so far. - total_passive_data: u32, + /// Record of all passive data segments that this module contains. + /// + /// These are processed during [`ModuleTranslation::finalize_memory_init`] + /// and eventually moved over into the `runtime_data` list above. Until + /// then, however, their `RuntimeDataIndex` is not yet assigned. + passive_data: Vec<(DataIndex, &'data [u8])>, /// When we're parsing the code section this will be incremented so we know /// which function is currently being defined. @@ -122,6 +115,62 @@ pub struct ModuleTranslation<'data> { /// The type information of the current module made available at the end of the /// validation process. types: Option, + + /// The WebAssembly `start` function, if defined. + pub start_func: Option, + + /// Initializers for `global` values which aren't considered "simple". + /// + /// These initializers are later compiled into a "module startup" function. + pub global_initializers: Vec<(DefinedGlobalIndex, ConstExpr)>, + + /// Definitions of all passive elements found within a module. + /// + /// This maps passive element segments to their definition, either functions + /// or expressions-basd. + pub passive_elements: PrimaryMap, + + /// WebAssembly table initialization data, per table. + /// + /// This keeps track of all per-table initialization (e.g. initial value for + /// non-null tables) as well as active element segments. This is processed + /// and refined by [`ModuleTranslation::finalize_table_init`] after + /// translation. + pub table_initialization: TableInitialization, + + /// WebAssembly memory initialization. + /// + /// This is held here in an `Unprocessed` form during translation, and then + /// this is later finished with [`ModuleTranslation::finalize_memory_init`]. + pub memory_init: MemoryInit<'data>, +} + +/// Different forms of memory initialization that happens for a module. +pub enum MemoryInit<'a> { + /// Raw active data segments that are being applied for an instance. + /// + /// This list contains the raw data which hasn't yet been processed into + /// `RuntimeDataIndex`, for example. This is later processed during + /// [`ModuleTranslation::finalize_memory_init`] to optionally shuffle things + /// around. + Unprocessed(Vec>), + + /// Finalized memory initialization to be executed after + /// [`ModuleTranslation::finalize_memory_init`] has run. This represents + /// active data segments which may have been merged from the `Unprocessed` + /// list above, and may or may not have statically know offsets. + Processed(Vec<(MemoryIndex, MemorySegmentOffset, RuntimeDataIndex)>), +} + +/// Offset within [`MemoryInit::Processed`] which indicates the initial offset +/// a data segment is applied at. +pub enum MemorySegmentOffset { + /// A "complicated" constant expression deferred to get evaluated at runtime + /// with compiled code. + Expr(ConstExpr), + + /// A statically known, in-bounds, constant value. + Static(u64), } impl<'data> ModuleTranslation<'data> { @@ -136,15 +185,18 @@ impl<'data> ModuleTranslation<'data> { exported_signatures: Vec::default(), debuginfo: DebugInfoData::default(), has_unparsed_debuginfo: false, - data: Vec::default(), data_align: None, - total_data: 0, - passive_data: Vec::default(), - total_passive_data: 0, + runtime_data: Default::default(), code_index: 0, types: None, - passive_data_map: Default::default(), + runtime_data_map: Default::default(), passive_elem_map: Default::default(), + start_func: None, + global_initializers: Vec::new(), + passive_elements: Default::default(), + table_initialization: Default::default(), + memory_init: MemoryInit::Unprocessed(Vec::new()), + passive_data: Default::default(), } } @@ -417,9 +469,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.result.module.needs_gc_heap |= table.ref_type.is_vmgcref_type(); self.result.module.tables.push(table)?; let init = match init { - wasmparser::TableInit::RefNull => TableInitialValue::Null { - precomputed: TryVec::new(), - }, + wasmparser::TableInit::RefNull => TableInitialValue::Null, wasmparser::TableInit::Expr(expr) => { let (init, escaped) = ConstExpr::from_wasmparser(self, expr)?; for f in escaped { @@ -428,11 +478,11 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { TableInitialValue::Expr(init) } }; + self.result.table_initialization.initial_values.push(init)?; self.result .module .table_initialization - .initial_values - .push(init)?; + .push(Default::default())?; } } @@ -475,8 +525,24 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.flag_func_escaped(f); } let ty = self.convert_global_type(&ty)?; - self.result.module.globals.push(ty)?; - self.result.module.global_initializers.push(initializer)?; + let index = self.result.module.globals.push(ty)?; + let defined_index = self.result.module.defined_global_index(index).unwrap(); + match initializer.const_eval() { + Some(val) => { + self.result + .module + .global_initializers + .push((defined_index, val))?; + } + None => { + // "Complicated" global initializers are deferred + // to get evaluated in the startup function. + self.require_startup_func(); + self.result + .global_initializers + .push((defined_index, initializer)); + } + } } } @@ -508,9 +574,12 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.validator.start_section(func, &range)?; let func_index = FuncIndex::from_u32(func); - self.flag_func_escaped(func_index); - debug_assert!(self.result.module.start_func.is_none()); - self.result.module.start_func = Some(func_index); + debug_assert!(self.result.start_func.is_none()); + self.result.start_func = Some(func_index); + + // To make startup a bit easier, invoking the `start` function + // is a responsibility deferred to the startup function. + self.require_startup_func(); } Payload::ElementSection(elements) => { @@ -566,19 +635,27 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?; debug_assert!(escaped.is_empty()); - self.result.module.table_initialization.segments.push( - TableSegment { + self.result + .table_initialization + .segments + .push(TableSegment { table_index, offset, elements, - }, - )?; + })?; None } ElementKind::Passive => { - let passive_index = - self.result.module.passive_elements.push(elements)?; + let passive_index = self + .result + .module + .passive_elements + .push((elements.ty(), elements.len()))?; + self.result.passive_elements.push(elements); + // One-time initialization of passive element + // segments is deferred to the startup function. + self.require_startup_func(); Some(passive_index) } @@ -639,14 +716,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { Payload::DataSection(data) => { self.validator.data_section(&data)?; - let initializers = match &mut self.result.module.memory_initialization { - MemoryInitialization::Segmented(i) => i, - _ => unreachable!(), - }; - - let cnt = usize::try_from(data.count()).unwrap(); - initializers.reserve_exact(cnt)?; - self.result.data.reserve_exact(cnt); + assert!(self.result.module.memory_initialization.is_segmented()); for (index, entry) in data.into_iter().enumerate() { let wasmparser::Data { @@ -655,53 +725,28 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { range: _, } = entry?; let data_index = DataIndex::from_u32(index.try_into().unwrap()); - let mk_range = |total: &mut u32| -> Result<_, WasmError> { - let range = u32::try_from(data.len()) - .ok() - .and_then(|size| { - let start = *total; - let end = start.checked_add(size)?; - Some(start..end) - }) - .ok_or_else(|| { - WasmError::Unsupported(format!( - "more than 4 gigabytes of data in wasm module", - )) - })?; - *total += range.end - range.start; - Ok(range) - }; - let passive_index = match kind { + match kind { DataKind::Active { memory_index, offset_expr, } => { - let range = mk_range(&mut self.result.total_data)?; let memory_index = MemoryIndex::from_u32(memory_index); let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?; debug_assert!(escaped.is_empty()); - let initializers = match &mut self.result.module.memory_initialization { - MemoryInitialization::Segmented(i) => i, - _ => unreachable!(), + let MemoryInit::Unprocessed(list) = &mut self.result.memory_init else { + panic!("memory initializers should be unprocessed at this point"); }; - initializers.push(MemoryInitializer { + list.push(MemoryInitializer { memory_index, offset, - data: range, - })?; - self.result.data.push(data.into()); - None + data, + }); } DataKind::Passive => { - let range = mk_range(&mut self.result.total_passive_data)?; - self.result.passive_data.push(data); - Some(self.result.module.passive_data.push(range)?) + self.result.passive_data.push((data_index, data)); } - }; - self.result - .passive_data_map - .insert(data_index, passive_index); + } } } @@ -952,6 +997,10 @@ and for re-adding support for interface types you can see this issue: } Ok(()) } + + fn require_startup_func(&mut self) { + self.result.require_startup_func(self.types); + } } impl TypeConvert for ModuleEnvironment<'_, '_> { @@ -971,6 +1020,75 @@ impl TypeConvert for ModuleEnvironment<'_, '_> { } impl ModuleTranslation<'_> { + /// Called after translation is complete this will finalize the memory + /// initialization strategy for this module. + /// + /// This will notably use `Self::try_static_init` to attempt to massage + /// data segments to being CoW-init-friendly. Afterwards the + /// `self.memory_init` field is transitioned from `Unprocessed` to + /// `Processed`. + pub fn finalize_memory_init( + &mut self, + tunables: &Tunables, + page_size: u64, + max_image_size_always_allowed: u64, + types: &mut ModuleTypesBuilder, + ) { + if tunables.memory_init_cow { + self.try_static_init(page_size, max_image_size_always_allowed); + } + + // If any memory is statically initialized, and if that memory has an + // initial data segment, then a startup function is at least + // conditionally needed if the memory needs initialization. Flag as such + // here. + if let MemoryInitialization::Static { map } = &self.module.memory_initialization { + if map.iter().any(|(_, v)| v.is_some()) { + self.require_startup_func_if_memories_need_init(types); + } + } + + // If, after `try_static_init`, initializers are still `Unprocessed` + // then this is the catch-all fallback path for initialization. All + // segments are promoted into `self.runtime_data` and then the + // initialization is rewritten to `Processed`. + if let MemoryInit::Unprocessed(list) = &mut self.memory_init { + let segments = mem::take(list); + let mut new_initializers = Vec::new(); + for segment in segments { + new_initializers.push(( + segment.memory_index, + MemorySegmentOffset::Expr(segment.offset), + self.runtime_data.push(segment.data.into()), + )); + } + if !new_initializers.is_empty() { + self.require_startup_func(types); + } + self.memory_init = MemoryInit::Processed(new_initializers); + } + + // At this point append all passive data to the `runtime_data` list. + // This notably occurs after `try_static_init` above to ensure that the + // page-aligned data for static initialization, if applicable, comes + // first. + for (data_index, segment) in self.passive_data.iter() { + let runtime_index = self.runtime_data.push((*segment).into()); + self.runtime_data_map + .insert(*data_index, Some(runtime_index)); + } + + // And, finally, record all chunks from `self.runtime_data` within + // `self.module.runtime_data` as well. + let mut cur = 0; + for (idx, data) in self.runtime_data.iter() { + let len = u32::try_from(data.len()).unwrap(); + let i = self.module.runtime_data.push(cur..cur + len).panic_on_oom(); + cur += len; + assert_eq!(idx, i); + } + } + /// Attempts to convert segmented memory initialization into static /// initialization for the module that this translation represents. /// @@ -1001,24 +1119,20 @@ impl ModuleTranslation<'_> { /// modules that have some dynamically-placed data segments. But, /// for now, this is sufficient to allow a system that "knows what /// it's doing" to always get static init. - pub fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) { - // This method only attempts to transform a `Segmented` memory init - // into a `Static` one, no other state. - if !self.module.memory_initialization.is_segmented() { - return; - } + fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) { + let segments = match &mut self.memory_init { + MemoryInit::Unprocessed(list) => list, + _ => return, + }; // First a dry run of memory initialization is performed. This // collects information about the extent of memory initialized for each // memory as well as the size of all data segments being copied in. - struct Memory { + struct Memory<'a> { data_size: u64, min_addr: u64, max_addr: u64, - // The `usize` here is a pointer into `self.data` which is the list - // of data segments corresponding to what was found in the original - // wasm module. - segments: Vec<(usize, StaticMemoryInitializer)>, + segments: Vec<(u64, &'a [u8])>, } let mut info = PrimaryMap::with_capacity(self.module.memories.len()); for _ in 0..self.module.memories.len() { @@ -1030,59 +1144,64 @@ impl ModuleTranslation<'_> { }); } - struct InitMemoryAtCompileTime<'a> { - module: &'a Module, - info: &'a mut PrimaryMap, - idx: usize, - } - impl InitMemory for InitMemoryAtCompileTime<'_> { - fn memory_size_in_bytes( - &mut self, - memory_index: MemoryIndex, - ) -> Result { - self.module.memories[memory_index].minimum_byte_size() + for initializer in segments.iter() { + let &MemoryInitializer { + memory_index, + ref offset, + ref data, + } = initializer; + + // Currently `Static` only applies to locally-defined memories, + // so if a data segment references an imported memory then + // transitioning to a `Static` memory initializer is not + // possible. + if self.module.defined_memory_index(memory_index).is_none() { + return; } - fn eval_offset(&mut self, memory_index: MemoryIndex, expr: &ConstExpr) -> Option { - match (expr.ops(), self.module.memories[memory_index].idx_type) { - (&[ConstOp::I32Const(offset)], IndexType::I32) => { - Some(offset.cast_unsigned().into()) + // First up determine the start/end range and verify that they're + // in-bounds for the initial size of the memory at `memory_index`. + // Note that this can bail if we don't have access to globals yet + // (e.g. this is a task happening before instantiation at + // compile-time). + let start = match (offset.ops(), self.module.memories[memory_index].idx_type) { + (&[ConstOp::I32Const(offset)], IndexType::I32) => offset.cast_unsigned().into(), + (&[ConstOp::I64Const(offset)], IndexType::I64) => offset.cast_unsigned(), + _ => return, + }; + let len = u64::try_from(data.len()).unwrap(); + let end = match start.checked_add(len) { + Some(end) => end, + None => return, + }; + + match self.module.memories[memory_index].minimum_byte_size() { + Ok(max) => { + if end > max { + return; } - (&[ConstOp::I64Const(offset)], IndexType::I64) => Some(offset.cast_unsigned()), - _ => None, } + + // Note that computing the minimum can overflow if the page + // size is the default 64KiB and the memory's minimum size in + // pages is `1 << 48`, the maximum number of minimum pages for + // 64-bit memories. We don't return `false` to signal an error + // here and instead defer the error to runtime, when it will be + // impossible to allocate that much memory anyways. + Err(_) => return, } - fn write(&mut self, memory: MemoryIndex, init: &StaticMemoryInitializer) -> bool { - // Currently `Static` only applies to locally-defined memories, - // so if a data segment references an imported memory then - // transitioning to a `Static` memory initializer is not - // possible. - if self.module.defined_memory_index(memory).is_none() { - return false; - }; - let info = &mut self.info[memory]; - let data_len = u64::from(init.data.end - init.data.start); - if data_len > 0 { - info.data_size += data_len; - info.min_addr = info.min_addr.min(init.offset); - info.max_addr = info.max_addr.max(init.offset + data_len); - info.segments.push((self.idx, init.clone())); - } - self.idx += 1; - true + // Skip empty in-bounds data segments. + if data.is_empty() { + continue; } - } - let ok = self - .module - .memory_initialization - .init_memory(&mut InitMemoryAtCompileTime { - idx: 0, - module: &self.module, - info: &mut info, - }); - if !ok { - return; + + let info = &mut info[memory_index]; + let len64 = u64::try_from(data.len()).unwrap(); + info.data_size += len64; + info.min_addr = info.min_addr.min(start); + info.max_addr = info.max_addr.max(start + len64); + info.segments.push((start, data)); } // Validate that the memory information collected is indeed valid for @@ -1127,9 +1246,8 @@ impl ModuleTranslation<'_> { // Here's where we've now committed to changing to static memory. The // memory initialization image is built here from the page data and then // it's converted to a single initializer. - let data = mem::replace(&mut self.data, Vec::new()); let mut map = TryPrimaryMap::with_capacity(info.len()).panic_on_oom(); - let mut module_data_size = 0u32; + let mut new_initializers = Vec::new(); for (memory, info) in info.iter() { // Create the in-memory `image` which is the initialized contents of // this linear memory. @@ -1139,10 +1257,8 @@ impl ModuleTranslation<'_> { 0 }; let mut image = Vec::with_capacity(extent); - for (idx, init) in info.segments.iter() { - let data = &data[*idx]; - assert_eq!(data.len(), init.data.len()); - let offset = usize::try_from(init.offset - info.min_addr).unwrap(); + for (offset, data) in info.segments.iter() { + let offset = usize::try_from(*offset - info.min_addr).unwrap(); if image.len() < offset { image.resize(offset, 0u8); image.extend_from_slice(data); @@ -1181,50 +1297,72 @@ impl ModuleTranslation<'_> { // the front and back with extra zeros as necessary if offset % page_size != 0 { let zero_padding = offset % page_size; - self.data.push(vec![0; zero_padding as usize].into()); + image.splice(0..0, std::iter::repeat(0).take(zero_padding as usize)); offset -= zero_padding; len += zero_padding; } - self.data.push(image.into()); if len % page_size != 0 { let zero_padding = page_size - (len % page_size); - self.data.push(vec![0; zero_padding as usize].into()); + image.extend(std::iter::repeat(0).take(zero_padding as usize)); len += zero_padding; } + let runtime_index = if image.is_empty() { + None + } else { + Some(self.runtime_data.push(image.into())) + }; // Offset/length should now always be page-aligned. assert!(offset % page_size == 0); assert!(len % page_size == 0); - // Create the `StaticMemoryInitializer` which describes this image, + // Record the static memory initializer which describes this image, // only needed if the image is actually present and has a nonzero // length. The `offset` has been calculates above, originally // sourced from `info.min_addr`. The `data` field is the extent // within the final data segment we'll emit to an ELF image, which // is the concatenation of `self.data`, so here it's the size of // the section-so-far plus the current segment we're appending. - let len = u32::try_from(len).unwrap(); - let init = if len > 0 { - Some(StaticMemoryInitializer { - offset, - data: module_data_size..module_data_size + len, - }) - } else { - None - }; - let idx = map.push(init).panic_on_oom(); + let idx = map.push(runtime_index.map(|i| (offset, i))).panic_on_oom(); assert_eq!(idx, memory); - module_data_size += len; + if let Some(runtime_index) = runtime_index { + new_initializers.push((idx, MemorySegmentOffset::Static(offset), runtime_index)); + } } self.data_align = Some(page_size); self.module.memory_initialization = MemoryInitialization::Static { map }; + self.memory_init = MemoryInit::Processed(new_initializers); + } + + /// Finalizes the initialization of tables. + /// + /// This is invoked after translation and notably uses + /// `Self::try_func_table_init` to attempt to optimize initialization of + /// tables into static precomputed images. + pub fn finalize_table_init(&mut self, tunables: &Tunables, types: &mut ModuleTypesBuilder) { + if tunables.table_lazy_init { + self.try_func_table_init(); + } + + // If any table has a non-null initializers, or if there's any active + // data segments, then a startup function is unconditionally required to + // configure the table. + if self + .table_initialization + .initial_values + .iter() + .any(|(_, v)| !matches!(v, TableInitialValue::Null)) + || !self.table_initialization.segments.is_empty() + { + self.require_startup_func(types); + } } /// Attempts to convert the module's table initializers to /// FuncTable form where possible. This enables lazy table /// initialization later by providing a one-to-one map of initial /// table values, without having to parse all segments. - pub fn try_func_table_init(&mut self) { + fn try_func_table_init(&mut self) { // This should be large enough to support very large Wasm // modules with huge funcref tables, but small enough to avoid // OOMs or DoS on truly sparse tables. @@ -1232,32 +1370,27 @@ impl ModuleTranslation<'_> { // First convert any element-initialized tables to images of just that // single function if the minimum size of the table allows doing so. - for ((_, init), (_, table)) in self - .module - .table_initialization - .initial_values - .iter_mut() - .zip( - self.module - .tables - .iter() - .skip(self.module.num_imported_tables), - ) - { + for ((i, init), (_, table)) in self.table_initialization.initial_values.iter_mut().zip( + self.module + .tables + .iter() + .skip(self.module.num_imported_tables), + ) { let table_size = table.limits.min; if table_size > MAX_FUNC_TABLE_SIZE { continue; } if let TableInitialValue::Expr(expr) = init { if let [ConstOp::RefFunc(f)] = expr.ops() { - *init = TableInitialValue::Null { - precomputed: try_vec![*f; table_size as usize].panic_on_oom(), - }; + assert!(self.module.table_initialization[i].is_empty()); + self.module.table_initialization[i] = + try_vec![*f; table_size as usize].panic_on_oom(); + *init = TableInitialValue::Null; } } } - let mut segments = mem::take(&mut self.module.table_initialization.segments) + let mut segments = mem::take(&mut self.table_initialization.segments) .into_iter() .peekable(); @@ -1326,18 +1459,18 @@ impl ModuleTranslation<'_> { TableSegmentElements::Expressions { .. } => break, }; - let precomputed = - match &mut self.module.table_initialization.initial_values[defined_index] { - TableInitialValue::Null { precomputed } => precomputed, + match &self.table_initialization.initial_values[defined_index] { + TableInitialValue::Null => {} - // If this table is still listed as an initial value here - // then that means the initial size of the table doesn't - // support a precomputed function list, so skip this. - // Technically this won't trap so it's possible to process - // further initializers, but that's left as a future - // optimization. - TableInitialValue::Expr(_) => break, - }; + // If this table is still listed as an initial value here + // then that means the initial size of the table doesn't + // support a precomputed function list, so skip this. + // Technically this won't trap so it's possible to process + // further initializers, but that's left as a future + // optimization. + TableInitialValue::Expr(_) => break, + } + let precomputed = &mut self.module.table_initialization[defined_index]; // At this point we're committing to pre-initializing the table // with the `segment` that's being iterated over. This segment is @@ -1355,6 +1488,27 @@ impl ModuleTranslation<'_> { // advance the iterator to see the next segment let _ = segments.next(); } - self.module.table_initialization.segments = segments.try_collect().panic_on_oom(); + self.table_initialization.segments = segments.try_collect().panic_on_oom(); + } + + /// Helper function to ratchet the `startup` function for this module as + /// `Always`. + fn require_startup_func(&mut self, types: &mut ModuleTypesBuilder) { + let ty = match self.module.startup { + ModuleStartup::None => types.startup_func_type().into(), + ModuleStartup::Always(_) => return, + ModuleStartup::IfMemoriesNeedInit(ty) => ty, + }; + self.module.startup = ModuleStartup::Always(ty); + } + + /// Helper function to ratchet the `startup` function for this module as + /// `IfMemoriesNeedInit`. + fn require_startup_func_if_memories_need_init(&mut self, types: &mut ModuleTypesBuilder) { + let ty = match self.module.startup { + ModuleStartup::None => types.startup_func_type().into(), + ModuleStartup::Always(_) | ModuleStartup::IfMemoriesNeedInit(_) => return, + }; + self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty); } } diff --git a/crates/environ/src/compile/module_types.rs b/crates/environ/src/compile/module_types.rs index a82843dc9ad7..082b3cab23f7 100644 --- a/crates/environ/src/compile/module_types.rs +++ b/crates/environ/src/compile/module_types.rs @@ -2,7 +2,7 @@ use crate::{ EngineOrModuleTypeIndex, EntityRef, ModuleInternedRecGroupIndex, ModuleInternedTypeIndex, ModuleTypes, PanicOnOom as _, TypeConvert, TypeIndex, WasmArrayType, WasmCompositeInnerType, WasmCompositeType, WasmExnType, WasmFieldType, WasmFuncType, WasmHeapType, WasmResult, - WasmStorageType, WasmStructType, WasmSubType, + WasmStorageType, WasmStructType, WasmSubType, WasmValType, collections::{TryClone as _, TryCow}, wasm_unsupported, }; @@ -59,6 +59,9 @@ pub struct ModuleTypesBuilder { /// If we are in the middle of defining a recursion group, this is the /// metadata about the recursion group we started defining. defining_rec_group: Option, + + /// Cache of the return value of [`Self::startup_func_type`]. + startup_func_type: Option, } impl ModuleTypesBuilder { @@ -72,6 +75,7 @@ impl ModuleTypesBuilder { wasmparser_to_wasmtime: HashMap::default(), already_seen: HashMap::default(), defining_rec_group: None, + startup_func_type: None, } } @@ -462,6 +466,45 @@ impl ModuleTypesBuilder { } Ok(composite_type.inner.unwrap_func()) } + + /// Gets a type for the "module startup" function, or `[] -> []` in the wasm + /// type system. + pub fn startup_func_type(&mut self) -> ModuleInternedTypeIndex { + *self.startup_func_type.get_or_insert_with(|| { + let idx = self.types.push(WasmSubType { + is_final: true, + supertype: None, + composite_type: WasmCompositeType { + inner: WasmCompositeInnerType::Func(WasmFuncType::new([], []).panic_on_oom()), + shared: false, + }, + }); + let next = self.types.next_ty(); + self.types.push_rec_group(idx..next); + idx + }) + } + + /// Smaller helper method to find a `ModuleInternedTypeIndex` which + /// corresponds to the `resource.drop` intrinsic in components, namely a + /// core wasm function type which takes one `i32` argument and has no + /// results. + /// + /// This is a bit of a hack right now as ideally this find operation + /// wouldn't be needed and instead the `ModuleInternedTypeIndex` itself + /// would be threaded through appropriately, but that's left for a future + /// refactoring. Try not to lean too hard on this method though. + pub fn find_resource_drop_signature(&self) -> Option { + self.wasm_types() + .find(|(_, ty)| { + ty.as_func().map_or(false, |sig| { + sig.params().len() == 1 + && sig.results().len() == 0 + && sig.params()[0] == WasmValType::I32 + }) + }) + .map(|(i, _)| i) + } } // Forward the indexing impl to the internal `ModuleTypes` diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs index 9e98fcac64a0..2bfc28a36145 100644 --- a/crates/environ/src/component/compiler.rs +++ b/crates/environ/src/component/compiler.rs @@ -10,7 +10,7 @@ pub trait ComponentCompiler: Send + Sync { /// Each trampoline is a member of the `Trampoline` enumeration and has a /// unique purpose and is translated differently. See the implementation of /// this trait for Cranelift for more information. - fn compile_trampoline( + fn compile_component_trampoline( &self, component: &ComponentTranslation, types: &ComponentTypesBuilder, diff --git a/crates/environ/src/component/types_builder.rs b/crates/environ/src/component/types_builder.rs index 3f8f7d94957d..d6cd8ec85bec 100644 --- a/crates/environ/src/component/types_builder.rs +++ b/crates/environ/src/component/types_builder.rs @@ -2,8 +2,8 @@ use crate::component::*; use crate::error::{Result, bail}; use crate::prelude::*; use crate::{ - EngineOrModuleTypeIndex, EntityType, ModuleInternedTypeIndex, ModuleTypes, ModuleTypesBuilder, - PrimaryMap, TypeConvert, WasmHeapType, WasmValType, + EngineOrModuleTypeIndex, EntityType, ModuleTypes, ModuleTypesBuilder, PrimaryMap, TypeConvert, + WasmHeapType, }; use cranelift_entity::EntityRef; use std::collections::HashMap; @@ -163,28 +163,6 @@ impl ComponentTypesBuilder { (self.component_types, ty) } - /// Smaller helper method to find a `ModuleInternedTypeIndex` which - /// corresponds to the `resource.drop` intrinsic in components, namely a - /// core wasm function type which takes one `i32` argument and has no - /// results. - /// - /// This is a bit of a hack right now as ideally this find operation - /// wouldn't be needed and instead the `ModuleInternedTypeIndex` itself - /// would be threaded through appropriately, but that's left for a future - /// refactoring. Try not to lean too hard on this method though. - pub fn find_resource_drop_signature(&self) -> Option { - self.module_types - .wasm_types() - .find(|(_, ty)| { - ty.as_func().map_or(false, |sig| { - sig.params().len() == 1 - && sig.results().len() == 0 - && sig.params()[0] == WasmValType::I32 - }) - }) - .map(|(i, _)| i) - } - /// Returns the underlying builder used to build up core wasm module types. /// /// Note that this is shared across all modules found within a component to diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rs index 8f0fd0e794ed..66bc7c189a8d 100644 --- a/crates/environ/src/key.rs +++ b/crates/environ/src/key.rs @@ -46,6 +46,10 @@ pub enum FuncKeyKind { /// A Wasmtime unsafe intrinsic function. #[cfg(feature = "component-model")] UnsafeIntrinsic = FuncKey::new_kind(0b1000), + + /// Initialization function for a module, such as initializing "complicated" + /// globals and passive element segments. + ModuleStartup = FuncKey::new_kind(0b1001), } impl From for u32 { @@ -73,6 +77,7 @@ impl FuncKeyKind { Self::PatchableToBuiltinTrampoline } x if x == Self::PulleyHostCall.into() => Self::PulleyHostCall, + x if x == Self::ModuleStartup.into() => Self::ModuleStartup, #[cfg(feature = "component-model")] x if x == Self::ComponentTrampoline.into() => Self::ComponentTrampoline, @@ -125,7 +130,9 @@ impl FuncKeyNamespace { /// Panics when given invalid raw representations. pub fn from_raw(raw: u32) -> Self { match FuncKeyKind::from_raw(raw & FuncKey::KIND_MASK) { - FuncKeyKind::DefinedWasmFunction | FuncKeyKind::ArrayToWasmTrampoline => Self(raw), + FuncKeyKind::DefinedWasmFunction + | FuncKeyKind::ArrayToWasmTrampoline + | FuncKeyKind::ModuleStartup => Self(raw), FuncKeyKind::WasmToArrayTrampoline | FuncKeyKind::WasmToBuiltinTrampoline | FuncKeyKind::PatchableToBuiltinTrampoline @@ -205,7 +212,6 @@ pub enum Abi { Patchable = 2, } -#[cfg(feature = "component-model")] impl Abi { fn from_raw(raw: u32) -> Self { match raw { @@ -254,6 +260,13 @@ pub enum FuncKey { /// A Wasmtime intrinsic function. #[cfg(feature = "component-model")] UnsafeIntrinsic(Abi, component::UnsafeIntrinsic), + + /// Initialization function for a module, such as initializing "complicated" + /// globals and passive element segments. + /// + /// This function has the `Abi` specified and will initialize the module + /// specified. + ModuleStartup(Abi, StaticModuleIndex), } impl Ord for FuncKey { @@ -342,6 +355,13 @@ impl FuncKey { let index = intrinsic.index(); (namespace, index) } + + FuncKey::ModuleStartup(abi, module) => { + assert_eq!(module.as_u32() & Self::KIND_MASK, 0); + let namespace = FuncKeyKind::ModuleStartup.into_raw() | module.as_u32(); + let index = abi.into_raw(); + (namespace, index) + } }; (FuncKeyNamespace(namespace), FuncKeyIndex(index)) } @@ -376,6 +396,7 @@ impl FuncKey { FuncKey::ResourceDropTrampoline => Abi::Wasm, #[cfg(feature = "component-model")] FuncKey::UnsafeIntrinsic(abi, _) => abi, + FuncKey::ModuleStartup(abi, _) => abi, } } @@ -462,6 +483,12 @@ impl FuncKey { let intrinsic = component::UnsafeIntrinsic::from_u32(b); Self::UnsafeIntrinsic(abi, intrinsic) } + + FuncKeyKind::ModuleStartup => { + let module = StaticModuleIndex::from_u32(a & Self::MODULE_MASK); + let abi = Abi::from_raw(b); + Self::ModuleStartup(abi, module) + } } } @@ -556,7 +583,9 @@ impl FuncKey { /// looking up the StoreCode (or having access to the Store). pub fn is_store_invariant(&self) -> bool { match self { - Self::DefinedWasmFunction(..) | Self::ArrayToWasmTrampoline(..) => false, + Self::DefinedWasmFunction(..) + | Self::ArrayToWasmTrampoline(..) + | Self::ModuleStartup(..) => false, Self::WasmToArrayTrampoline(..) | Self::WasmToBuiltinTrampoline(..) | Self::PatchableToBuiltinTrampoline(..) diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 1983b9729b74..45e228449bb6 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -7,8 +7,8 @@ use cranelift_entity::{EntityRef, packed_option::ReservedValue}; use serde_derive::{Deserialize, Serialize}; /// A WebAssembly linear memory initializer. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MemoryInitializer { +#[derive(Clone, Debug)] +pub struct MemoryInitializer<'a> { /// The index of a linear memory to initialize. pub memory_index: MemoryIndex, /// The base offset to start this segment at. @@ -17,19 +17,7 @@ pub struct MemoryInitializer { /// /// This range indexes into a separately stored data section which will be /// provided with the compiled module's code as well. - pub data: Range, -} - -/// Similar to the above `MemoryInitializer` but only used when memory -/// initializers are statically known to be valid. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct StaticMemoryInitializer { - /// The 64-bit offset, in bytes, of where this initializer starts. - pub offset: u64, - - /// The range of data to write at `offset`, where these indices are indexes - /// into the compiled wasm module's data section. - pub data: Range, + pub data: &'a [u8], } /// The type of WebAssembly linear memory initialization to use for a module. @@ -47,7 +35,7 @@ pub enum MemoryInitialization { /// data segments when the module is instantiated. /// /// This is the default memory initialization type. - Segmented(TryVec), + Segmented, /// Memory initialization is statically known and involves a single `memcpy` /// or otherwise simply making the defined data visible. @@ -81,13 +69,13 @@ pub enum MemoryInitialization { /// /// The offset, range base, and range end are all guaranteed to be page /// aligned to the page size passed in to `try_static_init`. - map: TryPrimaryMap>, + map: TryPrimaryMap>, }, } impl Default for MemoryInitialization { fn default() -> Self { - Self::Segmented(TryVec::new()) + Self::Segmented } } @@ -96,121 +84,10 @@ impl MemoryInitialization { /// `MemoryInitialization::Segmented`. pub fn is_segmented(&self) -> bool { match self { - MemoryInitialization::Segmented(_) => true, + MemoryInitialization::Segmented => true, _ => false, } } - - /// Performs the memory initialization steps for this set of initializers. - /// - /// This will perform wasm initialization in compliance with the wasm spec - /// and how data segments are processed. This doesn't need to necessarily - /// only be called as part of initialization, however, as it's structured to - /// allow learning about memory ahead-of-time at compile time possibly. - /// - /// This function will return true if all memory initializers are processed - /// successfully. If any initializer hits an error or, for example, a - /// global value is needed but `None` is returned, then false will be - /// returned. At compile-time this typically means that the "error" in - /// question needs to be deferred to runtime, and at runtime this means - /// that an invalid initializer has been found and a trap should be - /// generated. - pub fn init_memory(&self, state: &mut dyn InitMemory) -> bool { - let initializers = match self { - // Fall through below to the segmented memory one-by-one - // initialization. - MemoryInitialization::Segmented(list) => list, - - // If previously switched to static initialization then pass through - // all those parameters here to the `write` callback. - // - // Note that existence of `Static` already guarantees that all - // indices are in-bounds. - MemoryInitialization::Static { map } => { - for (index, init) in map { - if let Some(init) = init { - let result = state.write(index, init); - if !result { - return result; - } - } - } - return true; - } - }; - - for initializer in initializers { - let &MemoryInitializer { - memory_index, - ref offset, - ref data, - } = initializer; - - // First up determine the start/end range and verify that they're - // in-bounds for the initial size of the memory at `memory_index`. - // Note that this can bail if we don't have access to globals yet - // (e.g. this is a task happening before instantiation at - // compile-time). - let start = match state.eval_offset(memory_index, offset) { - Some(start) => start, - None => return false, - }; - let len = u64::try_from(data.len()).unwrap(); - let end = match start.checked_add(len) { - Some(end) => end, - None => return false, - }; - - match state.memory_size_in_bytes(memory_index) { - Ok(max) => { - if end > max { - return false; - } - } - - // Note that computing the minimum can overflow if the page size - // is the default 64KiB and the memory's minimum size in pages - // is `1 << 48`, the maximum number of minimum pages for 64-bit - // memories. We don't return `false` to signal an error here and - // instead defer the error to runtime, when it will be - // impossible to allocate that much memory anyways. - Err(_) => {} - } - - // The limits of the data segment have been validated at this point - // so the `write` callback is called with the range of data being - // written. Any erroneous result is propagated upwards. - let init = StaticMemoryInitializer { - offset: start, - data: data.clone(), - }; - let result = state.write(memory_index, &init); - if !result { - return result; - } - } - - return true; - } -} - -/// The various callbacks provided here are used to drive the smaller bits of -/// memory initialization. -pub trait InitMemory { - /// Returns the size, in bytes, of the memory specified. For compile-time - /// purposes this would be the memory type's minimum size. - fn memory_size_in_bytes(&mut self, memory_index: MemoryIndex) -> Result; - - /// Returns the value of the constant expression, as a `u64`. Note that - /// this may involve zero-extending a 32-bit global to a 64-bit number. May - /// return `None` to indicate that the expression involves a value which is - /// not available yet. - fn eval_offset(&mut self, memory_index: MemoryIndex, expr: &ConstExpr) -> Option; - - /// A callback used to actually write data. This indicates that the - /// specified memory must receive the specified range of data at the - /// specified offset. This can return false on failure. - fn write(&mut self, memory_index: MemoryIndex, init: &StaticMemoryInitializer) -> bool; } /// Table initialization data for all tables in the module. @@ -240,16 +117,7 @@ pub struct TableInitialization { pub enum TableInitialValue { /// Initialize each table element to null, optionally setting some elements /// to non-null given the precomputed image. - Null { - /// A precomputed image of table initializers for this table. - /// - /// This image is constructed during `try_func_table_init` and - /// null-initialized elements are represented with - /// `FuncIndex::reserved_value()`. Note that this image is empty by - /// default and may not encompass the entire span of the table in which - /// case the elements are initialized to null. - precomputed: TryVec, - }, + Null, /// An arbitrary const expression. Expr(ConstExpr), } @@ -285,6 +153,14 @@ pub enum TableSegmentElements { } impl TableSegmentElements { + /// Returns the type of this segment. + pub fn ty(&self) -> WasmRefType { + match self { + Self::Functions(_) => WasmRefType::FUNCREF, + Self::Expressions { ty, .. } => *ty, + } + } + /// Returns the number of elements in this segment. pub fn len(&self) -> u64 { match self { @@ -313,20 +189,38 @@ pub struct Module { /// Exported entities. pub exports: TryIndexMap, - /// The module "start" function, if present. - pub start_func: Option, + /// Whether or not this module has a start function, + pub startup: ModuleStartup, - /// WebAssembly table initialization data, per table. - pub table_initialization: TableInitialization, + /// Precompute per-table static images, if applicable. + /// + /// This map tracks, for each defined table in this module, the initial + /// precomputed contents of the table. This is only applicable for funcref + /// tables and the `TryVec` here uses `FuncIndex::reserved_value()` for null + /// entries. This structure is filled in if table initialization is detected + /// to be infallible as part of [`ModuleTranslation::finalize_table_init`]. + pub table_initialization: TryPrimaryMap>, /// WebAssembly linear memory initializer. + /// + /// This will track how memory is initialized, either exclusively via + /// segments or if some memories can be initialized with static images. This + /// is computed during [`ModuleTranslation::finalize_memory_init`]. pub memory_initialization: MemoryInitialization, /// WebAssembly passive elements. - pub passive_elements: TryPrimaryMap, + /// + /// This is a map of all passive element segments to their type and the + /// initial size of the segment. Note that the contents of the segment are + /// initialized by compiled code. + pub passive_elements: TryPrimaryMap, - /// Where passive data segments are located in the module's image. - pub passive_data: TryPrimaryMap>, + /// Where runtime data segments are located in the module's image. + /// + /// Note that this does not directly correspond to either active or passive + /// data segments. Those are massaged during + /// [`ModuleTranslation::finalize_memory_init`] into the form used here. + pub runtime_data: TryPrimaryMap>, /// Types declared in the wasm module. pub types: TryPrimaryMap, @@ -368,8 +262,18 @@ pub struct Module { /// WebAssembly global variables. pub globals: TryPrimaryMap, - /// WebAssembly global initializers for locally-defined globals. - pub global_initializers: TryPrimaryMap, + /// "Simple" WebAssembly global initializers for locally-defined globals. + /// + /// This map does not track initialization of all globals in this module, + /// but only those considered "simple" which can be easily evaluated at + /// compile-time. For example an initialization expression of `i32.const N` + /// is considered simple. These globals are manually initialized in the + /// host. + /// + /// This is all in contrast to [`ModuleTranslation::global_initializers`] + /// which is processed in compiled code and initialized after the instance + /// has been created. + pub global_initializers: TryVec<(DefinedGlobalIndex, GlobalConstValue)>, /// WebAssembly exception and control tags. pub tags: TryPrimaryMap, @@ -400,11 +304,11 @@ impl Module { name: Default::default(), initializers: Default::default(), exports: Default::default(), - start_func: Default::default(), + startup: ModuleStartup::None, table_initialization: Default::default(), memory_initialization: Default::default(), passive_elements: Default::default(), - passive_data: Default::default(), + runtime_data: Default::default(), types: Default::default(), num_imported_funcs: Default::default(), num_imported_tables: Default::default(), @@ -690,11 +594,11 @@ impl TypeTrace for Module { name: _, initializers: _, exports: _, - start_func: _, + startup, table_initialization: _, memory_initialization: _, passive_elements: _, - passive_data: _, + runtime_data: _, types, num_imported_funcs: _, num_imported_tables: _, @@ -726,6 +630,7 @@ impl TypeTrace for Module { for t in tags.values() { t.trace(func)?; } + startup.trace(func)?; Ok(()) } @@ -741,11 +646,11 @@ impl TypeTrace for Module { name: _, initializers: _, exports: _, - start_func: _, + startup, table_initialization: _, memory_initialization: _, passive_elements: _, - passive_data: _, + runtime_data: _, types, num_imported_funcs: _, num_imported_tables: _, @@ -777,10 +682,33 @@ impl TypeTrace for Module { for t in tags.values_mut() { t.trace_mut(func)?; } + startup.trace_mut(func)?; Ok(()) } } +impl TypeTrace for ModuleStartup { + fn trace(&self, func: &mut F) -> Result<(), E> + where + F: FnMut(EngineOrModuleTypeIndex) -> Result<(), E>, + { + match self { + ModuleStartup::None => Ok(()), + ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => func(*t), + } + } + + fn trace_mut(&mut self, func: &mut F) -> Result<(), E> + where + F: FnMut(&mut EngineOrModuleTypeIndex) -> Result<(), E>, + { + match self { + ModuleStartup::None => Ok(()), + ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => func(t), + } + } +} + /// Type information about functions in a wasm module. #[derive(Debug, Serialize, Deserialize)] pub struct FunctionType { @@ -821,3 +749,34 @@ impl FunctionType { #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] pub struct FuncRefIndex(u32); cranelift_entity::entity_impl!(FuncRefIndex); + +/// Different means of startup for a wasm module. +#[derive(Debug, Serialize, Deserialize)] +pub enum ModuleStartup { + /// No startup is necessary. + None, + + /// Startup is always required, for example to apply active table segments. + /// + /// The type of the startup function, of wasm signature `[] -> []`, is + /// provided here. + Always(EngineOrModuleTypeIndex), + + /// Startup is only required if some linear memory within this module, at + /// runtime, says `needs_init() == true`. + /// + /// This special mode of startup indicates that the startup function has no + /// purpose other than to initialize the initial contents of + /// `MemoryInitialization::Static` linear memories. In this situation if all + /// memories say `needs_init() == false` then the startup function won't + /// actually do anything meaning that it can be optimized slightly by + /// skipping it entirely. + IfMemoriesNeedInit(EngineOrModuleTypeIndex), +} + +impl ModuleStartup { + /// Returns if this is `ModuleStartup::None`. + pub fn is_none(&self) -> bool { + matches!(self, Self::None) + } +} diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs index 0f78e765cac5..c3173b3819e5 100644 --- a/crates/environ/src/module_artifacts.rs +++ b/crates/environ/src/module_artifacts.rs @@ -552,7 +552,8 @@ impl CompiledFunctionsTable { FuncKeyKind::ArrayToWasmTrampoline | FuncKeyKind::WasmToBuiltinTrampoline - | FuncKeyKind::PatchableToBuiltinTrampoline => false, + | FuncKeyKind::PatchableToBuiltinTrampoline + | FuncKeyKind::ModuleStartup => false, #[cfg(feature = "component-model")] FuncKeyKind::ComponentTrampoline @@ -569,7 +570,8 @@ impl CompiledFunctionsTable { | FuncKeyKind::WasmToArrayTrampoline | FuncKeyKind::WasmToBuiltinTrampoline | FuncKeyKind::PatchableToBuiltinTrampoline - | FuncKeyKind::PulleyHostCall => false, + | FuncKeyKind::PulleyHostCall + | FuncKeyKind::ModuleStartup => false, #[cfg(feature = "component-model")] FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rs index 0ee06f9378a3..9841f45ebcc2 100644 --- a/crates/environ/src/types.rs +++ b/crates/environ/src/types.rs @@ -1699,13 +1699,19 @@ impl Default for VMSharedTypeIndex { pub struct DataIndex(u32); entity_impl_with_try_clone!(DataIndex); -/// Index type of a passive data segment inside the WebAssembly module. +/// Index into data segments needed at runtime by a module. /// -/// Not a spec-level concept, just used to get dense index spaces for passive -/// data segments inside of Wasmtime. +/// This does not directly correspond to either active or passive data segments +/// in the wasm spec. Instead this is a concept purely for Wasmtime and +/// organizing memory initialization within the +/// `ModuleTranslation::finalize_memory_init` function, for example. +/// +/// Passive data segments at runtime all have a corresponding +/// `RuntimeDataIndex`, but active data segments maybe coalesced or mutated if +/// they're statically evaluated. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] -pub struct PassiveDataIndex(u32); -entity_impl_with_try_clone!(PassiveDataIndex); +pub struct RuntimeDataIndex(u32); +entity_impl_with_try_clone!(RuntimeDataIndex); /// Index type of an element segment inside the WebAssembly module. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] @@ -2007,7 +2013,7 @@ impl ConstExpr { /// A global's constant value, known at compile time. #[expect(missing_docs, reason = "self-describing variants")] -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub enum GlobalConstValue { I32(i32), I64(i64), @@ -2027,7 +2033,7 @@ pub enum ConstOp { V128Const(u128), GlobalGet(GlobalIndex), RefI31, - RefNull(WasmHeapTopType), + RefNull(WasmHeapType), RefFunc(FuncIndex), I32Add, I32Sub, @@ -2069,7 +2075,7 @@ impl ConstOp { O::F32Const { value } => Self::F32Const(value.bits()), O::F64Const { value } => Self::F64Const(value.bits()), O::V128Const { value } => Self::V128Const(u128::from_le_bytes(*value.bytes())), - O::RefNull { hty } => Self::RefNull(env.convert_heap_type(hty)?.top()), + O::RefNull { hty } => Self::RefNull(env.convert_heap_type(hty)?), O::RefFunc { function_index } => Self::RefFunc(FuncIndex::from_u32(function_index)), O::GlobalGet { global_index } => Self::GlobalGet(GlobalIndex::from_u32(global_index)), O::RefI31 => Self::RefI31, diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 74c50dd201e9..3437c1467426 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -31,13 +31,14 @@ // globals: [VMGlobalDefinition; module.num_defined_globals], // tags: [VMTagDefinition; module.num_defined_tags], // func_refs: [VMFuncRef; module.num_escaped_funcs], -// passive_data_bases: [*const u8; module.num_passive_data], -// passive_data_lengths: [u32; module.num_passive_data], +// startup_func_ref: [VMFuncRef; module.has_startup_func ? 1 : 0], +// runtime_data_bases: [*const u8; module.num_runtime_data], +// runtime_data_lengths: [u32; module.num_runtime_data], // } use crate::{ DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex, - FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, PassiveDataIndex, TableIndex, + FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex, TagIndex, }; use cranelift_entity::packed_option::ReservedValue; @@ -90,8 +91,10 @@ pub struct VMOffsets

{ /// The number of escaped functions in the module, the size of the func_refs /// array. pub num_escaped_funcs: u32, - /// The number of passive data segments in the module. - pub num_passive_data: u32, + /// The number of runtime data segments in the module. + pub num_runtime_data: u32, + /// Whether or not the module has a start function. + pub has_startup_func: bool, // precalculated offsets of various member fields imported_functions: u32, @@ -105,8 +108,9 @@ pub struct VMOffsets

{ defined_globals: u32, defined_tags: u32, defined_func_refs: u32, - passive_data_bases: u32, - passive_data_lengths: u32, + startup_func_ref: u32, + runtime_data_bases: u32, + runtime_data_lengths: u32, size: u32, } @@ -607,8 +611,10 @@ pub struct VMOffsetsFields

{ /// The number of escaped functions in the module, the size of the function /// references array. pub num_escaped_funcs: u32, - /// The number of passive data segments in the module. - pub num_passive_data: u32, + /// The number of runtime data segments in the module. + pub num_runtime_data: u32, + /// Whether or not the module has a start function. + pub has_startup_func: bool, } impl VMOffsets

{ @@ -635,7 +641,8 @@ impl VMOffsets

{ num_defined_globals: cast_to_u32(module.globals.len() - module.num_imported_globals), num_defined_tags: cast_to_u32(module.tags.len() - module.num_imported_tags), num_escaped_funcs: cast_to_u32(module.num_escaped_funcs), - num_passive_data: cast_to_u32(module.passive_data.len()), + num_runtime_data: cast_to_u32(module.runtime_data.len()), + has_startup_func: !module.startup.is_none(), }) } @@ -667,7 +674,8 @@ impl VMOffsets

{ num_defined_tags: _, num_owned_memories: _, num_escaped_funcs: _, - num_passive_data: _, + num_runtime_data: _, + has_startup_func: _, // used as the initial size below size, @@ -696,8 +704,9 @@ impl VMOffsets

{ } calculate_sizes! { - passive_data_lengths: "passive data lengths", - passive_data_bases: "passive data base pointers", + runtime_data_lengths: "runtime data lengths", + runtime_data_bases: "runtime data base pointers", + startup_func_ref: "startup funcref", defined_func_refs: "module functions", defined_tags: "defined tags", defined_globals: "defined globals", @@ -728,7 +737,8 @@ impl From> for VMOffsets

{ num_defined_globals: fields.num_defined_globals, num_defined_tags: fields.num_defined_tags, num_escaped_funcs: fields.num_escaped_funcs, - num_passive_data: fields.num_passive_data, + num_runtime_data: fields.num_runtime_data, + has_startup_func: fields.has_startup_func, imported_functions: 0, imported_tables: 0, imported_memories: 0, @@ -740,8 +750,9 @@ impl From> for VMOffsets

{ defined_globals: 0, defined_tags: 0, defined_func_refs: 0, - passive_data_bases: 0, - passive_data_lengths: 0, + startup_func_ref: 0, + runtime_data_bases: 0, + runtime_data_lengths: 0, size: 0, }; @@ -800,8 +811,13 @@ impl From> for VMOffsets

{ ret.num_escaped_funcs, ret.ptr.size_of_vm_func_ref(), ), - size(passive_data_bases) = cmul(ret.num_passive_data, ret.ptr.size()), - size(passive_data_lengths) = cmul(ret.num_passive_data, 4), + size(startup_func_ref) = if ret.has_startup_func { + ret.ptr.size_of_vm_func_ref() + } else { + 0 + }, + size(runtime_data_bases) = cmul(ret.num_runtime_data, ret.ptr.size()), + size(runtime_data_lengths) = cmul(ret.num_runtime_data, 4), } ret.size = next_field_offset; @@ -1050,16 +1066,16 @@ impl VMOffsets

{ self.defined_func_refs } - /// The offset of the `passive_data_bases` array. + /// The offset of the `runtime_data_bases` array. #[inline] - pub fn vmctx_passive_data_bases_begin(&self) -> u32 { - self.passive_data_bases + pub fn vmctx_runtime_data_bases_begin(&self) -> u32 { + self.runtime_data_bases } - /// The offset of the `passive_data_lengths` array. + /// The offset of the `runtime_data_lengths` array. #[inline] - pub fn vmctx_passive_data_lengths_begin(&self) -> u32 { - self.passive_data_lengths + pub fn vmctx_runtime_data_lengths_begin(&self) -> u32 { + self.runtime_data_lengths } /// Return the size of the `VMContext` allocation. @@ -1154,20 +1170,29 @@ impl VMOffsets

{ self.vmctx_func_refs_begin() + index.as_u32() * u32::from(self.ptr.size_of_vm_func_ref()) } - /// Return the offset to the base of the passive data segment at `index`. + /// Returns the offset to the `VMFuncRef` for the module startup function. + /// + /// Panics if this module does not have a startup function. + #[inline] + pub fn vmctx_startup_func_ref(&self) -> u32 { + assert!(self.has_startup_func); + self.startup_func_ref + } + + /// Return the offset to the base of the runtime data segment at `index`. #[inline] - pub fn vmctx_passive_data_base(&self, index: PassiveDataIndex) -> u32 { + pub fn vmctx_runtime_data_base(&self, index: RuntimeDataIndex) -> u32 { assert!(!index.is_reserved_value()); - assert!(index.as_u32() < self.num_passive_data); - self.vmctx_passive_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size()) + assert!(index.as_u32() < self.num_runtime_data); + self.vmctx_runtime_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size()) } - /// Return the offset to the length of the passive data segment at `index`. + /// Return the offset to the length of the runtime data segment at `index`. #[inline] - pub fn vmctx_passive_data_length(&self, index: PassiveDataIndex) -> u32 { + pub fn vmctx_runtime_data_length(&self, index: RuntimeDataIndex) -> u32 { assert!(!index.is_reserved_value()); - assert!(index.as_u32() < self.num_passive_data); - self.vmctx_passive_data_lengths_begin() + index.as_u32() * 4 + assert!(index.as_u32() < self.num_runtime_data); + self.vmctx_runtime_data_lengths_begin() + index.as_u32() * 4 } /// Return the offset to the `wasm_call` field in `*const VMFunctionBody` index `index`. diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs index 3696f325ab30..2994644c5d75 100644 --- a/crates/wasmtime/src/compile.rs +++ b/crates/wasmtime/src/compile.rs @@ -85,10 +85,11 @@ pub(crate) fn build_module_artifacts( ) .translate(parser, wasm) .context("failed to parse WebAssembly module")?; + prepare_translation(engine, compiler, &mut translation, &mut types); let functions = mem::take(&mut translation.function_body_inputs); let compile_inputs = CompileInputs::for_module(&types, &translation, functions); - let unlinked_compile_outputs = compile_inputs.compile(engine)?; + let unlinked_compile_outputs = compile_inputs.compile(engine, &types)?; let PreLinkOutput { needs_gc_heap, compiled_funcs, @@ -165,6 +166,15 @@ pub(crate) fn build_component_artifacts( .translate(binary) .context("failed to parse WebAssembly module")?; + for (_, translation) in module_translations.iter_mut() { + prepare_translation( + engine, + compiler, + translation, + types.module_types_builder_mut(), + ); + } + let compile_inputs = CompileInputs::for_component( engine, &types, @@ -174,7 +184,7 @@ pub(crate) fn build_component_artifacts( (i, &*translation, functions) }), ); - let unlinked_compile_outputs = compile_inputs.compile(&engine)?; + let unlinked_compile_outputs = compile_inputs.compile(&engine, types.module_types_builder())?; let PreLinkOutput { needs_gc_heap, @@ -230,6 +240,26 @@ pub(crate) fn build_component_artifacts( Ok((result, Some(artifacts))) } +fn prepare_translation( + engine: &Engine, + compiler: &dyn Compiler, + translation: &mut ModuleTranslation<'_>, + types: &mut ModuleTypesBuilder, +) { + // If configured attempt to use static memory initialization + // which can either at runtime be implemented as a single memcpy + // to initialize memory or otherwise enabling + // virtual-memory-tricks such as mmap'ing from a file to get + // copy-on-write. + let align = compiler.page_size_align(); + let max_always_allowed = engine.config().memory_guaranteed_dense_image_size; + translation.finalize_memory_init(engine.tunables(), align, max_always_allowed, types); + + // Attempt to convert table initializer segments to FuncTable + // representation where possible, to enable table lazy init. + translation.finalize_table_init(engine.tunables(), types); +} + type CompileInput<'a> = Box Result> + Send + 'a>; struct CompileOutput<'a> { @@ -357,7 +387,7 @@ impl<'a> CompileInputs<'a> { ); let function = compiler .component_compiler() - .compile_trampoline(component, types, key, abi, tunables, &symbol) + .compile_component_trampoline(component, types, key, abi, tunables, &symbol) .with_context(|| format!("failed to compile {symbol}"))?; Ok(CompileOutput { key, @@ -378,12 +408,16 @@ impl<'a> CompileInputs<'a> { // resources from the embedder and otherwise won't be explicitly // requested through initializers above or such. if component.component.num_resources > 0 { - if let Some(sig) = types.find_resource_drop_signature() { + if types + .module_types_builder() + .find_resource_drop_signature() + .is_some() + { ret.push_input(move |compiler| { let key = FuncKey::ResourceDropTrampoline; let symbol = "resource_drop_trampoline".to_string(); let function = compiler - .compile_wasm_to_array_trampoline(types[sig].unwrap_func(), key, &symbol) + .compile_trampoline(None, key, types.module_types_builder(), &symbol) .with_context(|| format!("failed to compile `{symbol}`"))?; Ok(CompileOutput { key, @@ -493,7 +527,7 @@ impl<'a> CompileInputs<'a> { func_index.as_u32() ); let function = compiler - .compile_array_to_wasm_trampoline(translation, types, key, &symbol) + .compile_trampoline(Some(translation), key, types, &symbol) .with_context(|| format!("failed to compile: {symbol}"))?; Ok(CompileOutput { key, @@ -506,6 +540,26 @@ impl<'a> CompileInputs<'a> { }); } } + + if !translation.module.startup.is_none() { + for abi in [Abi::Wasm, Abi::Array] { + self.push_input(move |compiler| { + let key = FuncKey::ModuleStartup(abi, module); + let symbol = format!("module_start[{}]::{abi:?}", module.as_u32()); + let function = compiler + .compile_trampoline(Some(translation), key, types, &symbol) + .with_context(|| format!("failed to compile: {symbol}"))?; + Ok(CompileOutput { + key, + function, + symbol, + start_srcloc: FilePos::default(), + translation: None, + func_body: None, + }) + }); + } + } } let mut trampoline_types_seen = HashSet::new(); @@ -514,7 +568,6 @@ impl<'a> CompileInputs<'a> { if !is_new { continue; } - let trampoline_func_ty = types[trampoline_type_index].unwrap_func(); self.push_input(move |compiler| { let key = FuncKey::WasmToArrayTrampoline(trampoline_type_index); let symbol = format!( @@ -522,7 +575,7 @@ impl<'a> CompileInputs<'a> { trampoline_type_index.as_u32() ); let function = compiler - .compile_wasm_to_array_trampoline(trampoline_func_ty, key, &symbol) + .compile_trampoline(None, key, types, &symbol) .with_context(|| format!("failed to compile: {symbol}"))?; Ok(CompileOutput { key, @@ -538,7 +591,11 @@ impl<'a> CompileInputs<'a> { /// Compile these `CompileInput`s (maybe in parallel) and return the /// resulting `UnlinkedCompileOutput`s. - fn compile(self, engine: &Engine) -> Result> { + fn compile( + self, + engine: &Engine, + types: &'a ModuleTypesBuilder, + ) -> Result> { let compiler = engine.try_compiler()?; if self.inputs.len() > 0 && cfg!(miri) { @@ -597,7 +654,7 @@ the use case. // wasmtime-builtin functions are necessary. If so those need to be // collected and then those trampolines additionally need to be // compiled. - compile_required_builtins(engine, &mut raw_outputs)?; + compile_required_builtins(engine, types, &mut raw_outputs)?; // Bucket the outputs by kind. let mut outputs: BTreeMap = BTreeMap::new(); @@ -910,7 +967,8 @@ fn is_inlining_function(key: FuncKey) -> bool { FuncKey::ArrayToWasmTrampoline(..) | FuncKey::WasmToArrayTrampoline(..) | FuncKey::WasmToBuiltinTrampoline(..) - | FuncKey::PatchableToBuiltinTrampoline(..) => false, + | FuncKey::PatchableToBuiltinTrampoline(..) + | FuncKey::ModuleStartup(..) => false, #[cfg(feature = "component-model")] FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false, @@ -934,7 +992,11 @@ fn inlining_functions<'a>( }) } -fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec) -> Result<()> { +fn compile_required_builtins<'a>( + engine: &Engine, + types: &'a ModuleTypesBuilder, + raw_outputs: &mut Vec>, +) -> Result<()> { let compiler = engine.try_compiler()?; let mut builtins = HashSet::new(); let mut new_inputs: Vec> = Vec::new(); @@ -951,7 +1013,7 @@ fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec unreachable!(), }; let mut function = compiler - .compile_wasm_to_builtin(key, &symbol) + .compile_trampoline(None, key, types, &symbol) .with_context(|| format!("failed to compile `{symbol}`"))?; if let Some(compiler) = compiler.inlining_compiler() { compiler.finish_compiling(&mut function, None, &symbol)?; @@ -1116,26 +1178,7 @@ impl FunctionIndices { let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables); let modules = translations .into_iter() - .map(|(_, mut translation)| { - // If configured attempt to use static memory initialization - // which can either at runtime be implemented as a single memcpy - // to initialize memory or otherwise enabling - // virtual-memory-tricks such as mmap'ing from a file to get - // copy-on-write. - if engine.tunables().memory_init_cow { - let align = compiler.page_size_align(); - let max_always_allowed = engine.config().memory_guaranteed_dense_image_size; - translation.try_static_init(align, max_always_allowed); - } - - // Attempt to convert table initializer segments to FuncTable - // representation where possible, to enable table lazy init. - if engine.tunables().table_lazy_init { - translation.try_func_table_init(); - } - - obj.append(translation) - }) + .map(|(_, translation)| obj.append(translation)) .collect::>>()?; let artifacts = Artifacts { diff --git a/crates/wasmtime/src/runtime/code.rs b/crates/wasmtime/src/runtime/code.rs index 3d6af317c3fb..1faf0ac073fa 100644 --- a/crates/wasmtime/src/runtime/code.rs +++ b/crates/wasmtime/src/runtime/code.rs @@ -9,11 +9,9 @@ use alloc::boxed::Box; use alloc::sync::Arc; use core::ops::{Add, Range, Sub}; use wasmtime_core::error::OutOfMemory; -use wasmtime_environ::DefinedFuncIndex; -use wasmtime_environ::ModuleTypes; -use wasmtime_environ::StaticModuleIndex; #[cfg(feature = "component-model")] use wasmtime_environ::component::ComponentTypes; +use wasmtime_environ::{FuncKey, ModuleTypes, StaticModuleIndex}; macro_rules! define_pc_kind { ($ty:ident) => { @@ -431,30 +429,10 @@ impl<'a> ModuleWithCode<'a> { } /// Returns the slice in the text section of the function that - /// `index` points to. + /// `key` points to. #[inline] - pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] { - let range = self - .module - .compiled_module() - .finished_function_range(def_func_index); + pub fn function(&self, key: FuncKey) -> &[u8] { + let range = self.module.compiled_module().function_range(key); &self.store_code.text()[range] } - - /// Get the array-to-Wasm trampoline for the function `index` - /// points to, as a slice of raw code that can be converted to a - /// callable function pointer. - /// - /// If the function `index` points to does not escape, then `None` is - /// returned. - /// - /// These trampolines are used for array callers (e.g. `Func::new`) - /// calling Wasm callees. - pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> { - let range = self - .module - .compiled_module() - .array_to_wasm_trampoline_range(def_func_index)?; - Some(&self.store_code.text()[range]) - } } diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rs index 489caf0a8e3d..7b4f66cee674 100644 --- a/crates/wasmtime/src/runtime/component/component.rs +++ b/crates/wasmtime/src/runtime/component/component.rs @@ -967,6 +967,7 @@ mod tests { use wasmtime_environ::MemoryInitialization; #[test] + #[cfg_attr(miri, ignore)] fn cow_on_by_default() { let mut config = Config::new(); config.wasm_component_model(true); diff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs index d03ba277f12b..eb2f392270ac 100644 --- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs +++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs @@ -7,7 +7,7 @@ use core::convert::Infallible; use core::fmt::{self, Debug}; use core::hash::{Hash, Hasher}; use core::marker; -use core::ops::{Deref, DerefMut}; +use core::ops::Deref; mod sealed { use super::*; @@ -209,27 +209,3 @@ impl RootedGcRefImpl for OwnedRooted { match self.inner {} } } - -pub(crate) struct OpaqueRootScope { - store: S, -} - -impl Deref for OpaqueRootScope { - type Target = S; - - fn deref(&self) -> &Self::Target { - &self.store - } -} - -impl DerefMut for OpaqueRootScope { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.store - } -} - -impl OpaqueRootScope { - pub(crate) fn new(store: S) -> Self { - OpaqueRootScope { store } - } -} diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs index f2133cc2759e..1b4a53971020 100644 --- a/crates/wasmtime/src/runtime/instance.rs +++ b/crates/wasmtime/src/runtime/instance.rs @@ -14,7 +14,6 @@ use crate::{ }; use alloc::sync::Arc; use core::ptr::NonNull; -use wasmparser::WasmFeatures; use wasmtime_environ::{ EntityIndex, EntityType, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TagIndex, TypeTrace, }; @@ -255,21 +254,26 @@ impl Instance { imports: Imports<'_>, asyncness: Asyncness, ) -> Result { - let (instance, start) = { + let instance = { let (mut limiter, store) = store.0.resource_limiter_and_store_opaque(); // SAFETY: the safety contract of `new_raw` is the same as this // function. - unsafe { Instance::new_raw(store, limiter.as_mut(), module, imports, asyncness).await? } + unsafe { Instance::new_raw(store, limiter.as_mut(), module, imports).await? } }; - if let Some(start) = start { + + // If this instance requires startup, which is a dynamic decision made + // at this point in conjunction with analysis at compile time, the + // instance gets started. Note that this isn't just the wasm start + // function itself, but it's finalization of initialization of this + // instance, for example for complicated global initialization + // expressions. + if instance.id.get_mut(store.0).needs_startup() { if asyncness == Asyncness::No { - instance.start_raw(store, start)?; + instance.start_raw(store)?; } else { #[cfg(feature = "async")] { - store - .on_fiber(|store| instance.start_raw(store, start)) - .await??; + store.on_fiber(|store| instance.start_raw(store)).await??; } #[cfg(not(feature = "async"))] unreachable!(); @@ -298,8 +302,7 @@ impl Instance { mut limiter: Option<&mut StoreResourceLimiter<'_>>, module: &Module, imports: Imports<'_>, - asyncness: Asyncness, - ) -> Result<(Instance, Option)> { + ) -> Result { if !Engine::same(store.engine(), module.engine()) { bail!("cross-`Engine` instantiation is not currently supported"); } @@ -317,8 +320,6 @@ impl Instance { } } - let compiled_module = module.compiled_module(); - // Register the module just before instantiation to ensure we keep the module // properly referenced while in use by the store. let (modules, engine, breakpoints) = store.modules_and_engine_and_breakpoints_mut(); @@ -341,46 +342,12 @@ impl Instance { .await? }; - // Additionally, before we start doing fallible instantiation, we - // do one more step which is to insert an `InstanceData` - // corresponding to this instance. This `InstanceData` can be used - // via `Caller::get_export` if our instance's state "leaks" into - // other instances, even if we don't return successfully from this - // function. - // - // We don't actually load all exports from the instance at this - // time, instead preferring to lazily load them as they're demanded. - // For module/instance exports, though, those aren't actually - // stored in the instance handle so we need to immediately handle - // those here. - let instance = Instance::from_wasmtime(id, store); - - // Now that we've recorded all information we need to about this - // instance within a `Store` we can start performing fallible - // initialization. Note that we still defer the `start` function to - // later since that may need to run asynchronously. - // - // If this returns an error (or if the start function traps) then - // any other initialization which may have succeeded which placed - // items from this instance into other instances should be ok when - // those items are loaded and run we'll have all the metadata to - // look at them. - let bulk_memory = store - .engine() - .features() - .contains(WasmFeatures::BULK_MEMORY); - - vm::initialize_instance( - store, - limiter, - id, - compiled_module.module(), - bulk_memory, - asyncness, - ) - .await?; - - Ok((instance, compiled_module.module().start_func)) + // At this point the instance is created and stored within the store, + // but it's also not quite usable just yet. Initialization hasn't + // completed (e.g. active data/element segments) and the `start` + // function additionally has not yet been invoked. That's the + // responsibility of the caller to handle, however. + Ok(Instance::from_wasmtime(id, store)) } pub(crate) fn from_wasmtime(id: InstanceId, store: &mut StoreOpaque) -> Instance { @@ -389,7 +356,7 @@ impl Instance { } } - fn start_raw(&self, store: &mut StoreContextMut<'_, T>, start: FuncIndex) -> Result<()> { + fn start_raw(&self, store: &mut StoreContextMut<'_, T>) -> Result<()> { // If a start function is present, invoke it. Make sure we use all the // trap-handling configuration in `store` as well. let store_id = store.0.id(); @@ -399,7 +366,8 @@ impl Instance { let f = unsafe { instance .as_mut() - .get_exported_func(registry, store_id, start) + .get_startup_func(registry, store_id) + .expect("should have a startup function") }; let caller_vmctx = instance.vmctx(); unsafe { diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rs index 188d005e1fd7..8640cdf43acf 100644 --- a/crates/wasmtime/src/runtime/instantiate.rs +++ b/crates/wasmtime/src/runtime/instantiate.rs @@ -11,9 +11,8 @@ use alloc::sync::Arc; use core::ops::Range; use core::str; use wasmtime_environ::{ - CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, EntityRef, FilePos, FuncIndex, - FuncKey, FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex, - StaticModuleIndex, + CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, FilePos, FuncIndex, FuncKey, + FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex, StaticModuleIndex, }; /// A compiled wasm module, ready to be instantiated. @@ -116,55 +115,32 @@ impl CompiledModule { pub fn finished_function_ranges( &self, ) -> impl ExactSizeIterator)> + '_ { - self.module - .defined_func_indices() - .map(|i| (i, self.finished_function_range(i))) + self.module.defined_func_indices().map(|i| { + let key = FuncKey::DefinedWasmFunction(self.module_index(), i); + (i, self.function_range(key)) + }) } - /// Returns the offset in the text section of the function that `index` points to. + /// Returns the offset in the text section of the function that `key` + /// points to. #[inline] - pub fn finished_function_range(&self, def_func_index: DefinedFuncIndex) -> Range { - let loc = self.func_loc(def_func_index); + pub fn function_range(&self, key: FuncKey) -> Range { + let loc = self.func_loc(key); let start = usize::try_from(loc.start).unwrap(); let end = usize::try_from(loc.start + loc.length).unwrap(); start..end } - /// Get the array-to-Wasm trampoline for the function `index` - /// points to, as a range in the text segment. - /// - /// If the function `index` points to does not escape, then `None` is - /// returned. - /// - /// These trampolines are used for array callers (e.g. `Func::new`) - /// calling Wasm callees. - pub fn array_to_wasm_trampoline_range( - &self, - def_func_index: DefinedFuncIndex, - ) -> Option> { - assert!(def_func_index.index() < self.module.num_defined_funcs()); - let key = FuncKey::ArrayToWasmTrampoline(self.module_index(), def_func_index); - let loc = self.index.func_loc(key)?; - let start = usize::try_from(loc.start).unwrap(); - let end = usize::try_from(loc.start + loc.length).unwrap(); - Some(start..end) - } - /// Get the Wasm-to-array trampoline for the given signature, as a /// range in the text segment. /// /// These trampolines are used for filling in /// `VMFuncRef::wasm_call` for `Func::wrap`-style host funcrefs /// that don't have access to a compiler when created. - pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> Option<&[u8]> { + pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> &[u8] { let key = FuncKey::WasmToArrayTrampoline(signature); - let loc = self.index.func_loc(key)?; - let start = usize::try_from(loc.start).unwrap(); - let end = usize::try_from(loc.start + loc.length).unwrap(); - Some( - self.engine_code - .raw_wasm_to_array_trampoline_data(start..end), - ) + let range = self.function_range(key); + self.engine_code.raw_wasm_to_array_trampoline_data(range) } /// Lookups a defined function by a program counter value. @@ -190,9 +166,7 @@ impl CompiledModule { } /// Gets the function location information for a given function index. - pub fn func_loc(&self, def_func_index: DefinedFuncIndex) -> &FunctionLoc { - assert!(def_func_index.index() < self.module.num_defined_funcs()); - let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index); + pub fn func_loc(&self, key: FuncKey) -> &FunctionLoc { self.index .func_loc(key) .expect("defined function should be present") @@ -200,9 +174,7 @@ impl CompiledModule { /// Returns the original binary offset in the file that `index` was defined /// at. - pub fn func_start_srcloc(&self, def_func_index: DefinedFuncIndex) -> FilePos { - assert!(def_func_index.index() < self.module.num_defined_funcs()); - let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index); + pub fn func_start_srcloc(&self, key: FuncKey) -> FilePos { self.index .src_loc(key) .expect("defined function should be present") diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rs index 9b4294f34fcf..395225580417 100644 --- a/crates/wasmtime/src/runtime/module.rs +++ b/crates/wasmtime/src/runtime/module.rs @@ -19,8 +19,8 @@ use core::ptr::NonNull; use std::{fs::File, path::Path}; use wasmparser::{Parser, ValidPayload, Validator}; use wasmtime_environ::{ - CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind, - StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum, + CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, FuncKey, HostPtr, ModuleTypes, + ObjectKind, StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum, }; mod registry; @@ -1092,7 +1092,8 @@ impl Module { let module = self.compiled_module(); let module_index = self.env_module().module_index; self.env_module().defined_func_indices().map(move |idx| { - let loc = module.func_loc(idx); + let key = FuncKey::DefinedWasmFunction(module_index, idx); + let loc = module.func_loc(key); let idx = module.module().func_index(idx); ModuleFunction { module: module_index, @@ -1158,7 +1159,6 @@ impl Module { let ptr = self .compiled_module() .wasm_to_array_trampoline(trampoline_module_ty) - .expect("always have a trampoline for the trampoline type") .as_ptr() .cast::() .cast_mut(); @@ -1275,6 +1275,7 @@ mod tests { use wasmtime_environ::MemoryInitialization; #[test] + #[cfg_attr(miri, ignore)] fn cow_on_by_default() { let engine = Engine::default(); let module = Module::new( diff --git a/crates/wasmtime/src/runtime/profiling.rs b/crates/wasmtime/src/runtime/profiling.rs index 30234e32bcbe..336639c7983f 100644 --- a/crates/wasmtime/src/runtime/profiling.rs +++ b/crates/wasmtime/src/runtime/profiling.rs @@ -13,7 +13,7 @@ use fxprof_processed_profile::{ use std::ops::Range; use std::sync::Arc; use std::time::{Duration, Instant}; -use wasmtime_environ::demangle_function_name_or_index; +use wasmtime_environ::{FuncKey, demangle_function_name_or_index}; // TODO: collect more data // - On non-Windows, measure thread-local CPU usage between events with @@ -268,7 +268,9 @@ fn module_symbols(name: String, module: &Module) -> Option { .env_module() .defined_func_indices() .map(|defined_idx| { - let loc = compiled.func_loc(defined_idx); + let key = + FuncKey::DefinedWasmFunction(module.env_module().module_index, defined_idx); + let loc = compiled.func_loc(key); let func_idx = compiled.module().func_index(defined_idx); let mut name = String::new(); demangle_function_name_or_index( diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rs index cc49f4a7cf6f..e7d79cb4bf99 100644 --- a/crates/wasmtime/src/runtime/trap.rs +++ b/crates/wasmtime/src/runtime/trap.rs @@ -7,7 +7,7 @@ use core::fmt; use core::num::NonZeroUsize; use wasmtime_core::alloc::TryVec; use wasmtime_environ::{ - CompiledTrap, FilePos, demangle_function_name, demangle_function_name_or_index, + CompiledTrap, FilePos, FuncKey, demangle_function_name, demangle_function_name_or_index, }; /// Representation of a WebAssembly trap and what caused it to occur. @@ -501,7 +501,8 @@ impl FrameInfo { pub(crate) fn new(module: Module, text_offset: usize) -> Option { let compiled_module = module.compiled_module(); let index = compiled_module.func_by_text_offset(text_offset)?; - let func_start = compiled_module.func_start_srcloc(index); + let key = FuncKey::DefinedWasmFunction(compiled_module.module().module_index, index); + let func_start = compiled_module.func_start_srcloc(key); let instr = wasmtime_environ::lookup_file_pos(module.engine_code().address_map_data(), text_offset); let index = compiled_module.module().func_index(index); diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs index 832652a6949a..dd2830084df3 100644 --- a/crates/wasmtime/src/runtime/values.rs +++ b/crates/wasmtime/src/runtime/values.rs @@ -4,7 +4,6 @@ use crate::{ StructRef, V128, ValType, prelude::*, }; use core::ptr; -use wasmtime_environ::WasmHeapTopType; pub use crate::runtime::vm::ValRaw; @@ -125,16 +124,6 @@ impl Val { Val::AnyRef(None) } - pub(crate) const fn null_top(top: WasmHeapTopType) -> Val { - match top { - WasmHeapTopType::Func => Val::FuncRef(None), - WasmHeapTopType::Extern => Val::ExternRef(None), - WasmHeapTopType::Any => Val::AnyRef(None), - WasmHeapTopType::Exn => Val::ExnRef(None), - WasmHeapTopType::Cont => Val::ContRef(None), - } - } - /// Returns the default value for the given type, if any exists. /// /// Returns `None` if there is no default value for the given type (for diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 3a57c66547f3..a42ec6666bd6 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -53,7 +53,6 @@ use wasmtime_environ::ModuleInternedTypeIndex; mod always_mut; #[cfg(feature = "component-model")] pub mod component; -mod const_expr; mod export; mod gc; mod imports; @@ -100,7 +99,7 @@ pub use crate::runtime::vm::gc::*; pub use crate::runtime::vm::imports::Imports; pub use crate::runtime::vm::instance::{ GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, - MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, initialize_instance, + MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, }; #[cfg(feature = "pooling-allocator")] pub use crate::runtime::vm::instance::{ diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rs deleted file mode 100644 index a3d836bacbf9..000000000000 --- a/crates/wasmtime/src/runtime/vm/const_expr.rs +++ /dev/null @@ -1,499 +0,0 @@ -//! Evaluating const expressions. - -use crate::prelude::*; -use crate::runtime::vm; -use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter}; -#[cfg(feature = "gc")] -use crate::{ - AnyRef, ArrayRef, ArrayRefPre, ArrayType, ExternRef, I31, StructRef, StructRefPre, StructType, -}; -use crate::{OpaqueRootScope, Val}; -use wasmtime_environ::{ConstExpr, ConstOp, FuncIndex, GlobalConstValue, GlobalIndex}; -#[cfg(feature = "gc")] -use wasmtime_environ::{VMSharedTypeIndex, WasmCompositeInnerType, WasmCompositeType, WasmSubType}; - -/// An interpreter for const expressions. -/// -/// This can be reused across many const expression evaluations to reuse -/// allocated resources, if any. -pub struct ConstExprEvaluator { - stack: TryVec, - simple: Val, -} - -impl Default for ConstExprEvaluator { - fn default() -> ConstExprEvaluator { - ConstExprEvaluator { - stack: TryVec::new(), - simple: Val::I32(0), - } - } -} - -/// The context within which a particular const expression is evaluated. -pub struct ConstEvalContext { - pub(crate) instance: InstanceId, - #[cfg_attr( - not(feature = "gc"), - expect(dead_code, reason = "easier than conditionally compiling this field") - )] - pub(crate) asyncness: Asyncness, -} - -impl ConstEvalContext { - /// Create a new context. - pub fn new(instance: InstanceId, asyncness: Asyncness) -> Self { - Self { - instance, - asyncness, - } - } - - fn global_get(&mut self, store: &mut StoreOpaque, index: GlobalIndex) -> Result { - let id = store.id(); - Ok(store - .instance_mut(self.instance) - .get_exported_global(id, index) - ._get(&mut AutoAssertNoGc::new(store))) - } - - fn ref_func(&mut self, store: &mut StoreOpaque, index: FuncIndex) -> Result { - let id = store.id(); - let (instance, registry) = store.instance_and_module_registry_mut(self.instance); - // SAFETY: `id` is the correct store-owner of the function being looked - // up - let func = unsafe { instance.get_exported_func(registry, id, index) }; - Ok(func.into()) - } - - #[cfg(feature = "gc")] - fn struct_fields_len(&self, store: &mut StoreOpaque, shared_ty: VMSharedTypeIndex) -> usize { - let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty); - let fields = struct_ty.fields(); - fields.len() - } - - #[cfg(feature = "gc")] - async fn struct_new( - &mut self, - store: &mut StoreOpaque, - limiter: Option<&mut StoreResourceLimiter<'_>>, - shared_ty: VMSharedTypeIndex, - fields: &[Val], - ) -> Result { - let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty); - let allocator = StructRefPre::_new(store, struct_ty); - let struct_ref = - StructRef::_new_async(store, limiter, &allocator, &fields, self.asyncness).await?; - Ok(Val::AnyRef(Some(struct_ref.into()))) - } - - #[cfg(feature = "gc")] - async fn struct_new_default( - &mut self, - store: &mut StoreOpaque, - limiter: Option<&mut StoreResourceLimiter<'_>>, - shared_ty: VMSharedTypeIndex, - ) -> Result { - let module = store - .instance(self.instance) - .runtime_module() - .expect("should never be allocating a struct type defined in a dummy module"); - - let borrowed = module - .engine() - .signatures() - .borrow(shared_ty) - .expect("should have a registered type for struct"); - let WasmSubType { - composite_type: - WasmCompositeType { - shared: false, - inner: WasmCompositeInnerType::Struct(struct_ty), - }, - .. - } = &*borrowed - else { - unreachable!("registered type should be a struct"); - }; - - let fields = struct_ty - .fields - .iter() - .map(|ty| match &ty.element_type { - wasmtime_environ::WasmStorageType::I8 | wasmtime_environ::WasmStorageType::I16 => { - Val::I32(0) - } - wasmtime_environ::WasmStorageType::Val(v) => match v { - wasmtime_environ::WasmValType::I32 => Val::I32(0), - wasmtime_environ::WasmValType::I64 => Val::I64(0), - wasmtime_environ::WasmValType::F32 => Val::F32(0.0f32.to_bits()), - wasmtime_environ::WasmValType::F64 => Val::F64(0.0f64.to_bits()), - wasmtime_environ::WasmValType::V128 => Val::V128(0u128.into()), - wasmtime_environ::WasmValType::Ref(r) => { - assert!(r.nullable); - Val::null_top(r.heap_type.top()) - } - }, - }) - .collect::>(); - - self.struct_new(store, limiter, shared_ty, &fields).await - } -} - -impl ConstExprEvaluator { - /// Same as [`Self::eval`] except that this is specifically intended for - /// integral constant expression. - /// - /// # Panics - /// - /// Panics if `ConstExpr` contains GC ops (e.g. it's not for an integral - /// type). - pub fn eval_int( - &mut self, - store: &mut StoreOpaque, - context: &mut ConstEvalContext, - expr: &ConstExpr, - ) -> Result<&Val> { - // Try to evaluate a simple expression first before doing the more - // complicated eval loop below. - if self.try_simple(expr).is_some() { - return Ok(&self.simple); - } - - // Note that `assert_ready` here should be valid as production of an - // integer cannot involve GC meaning that async operations aren't used. - let mut scope = OpaqueRootScope::new(store); - vm::assert_ready(self.eval_loop(&mut scope, None, context, expr)) - } - - /// Attempts to peek into `expr` to see if it's trivial to evaluate, e.g. - /// for `i32.const N`. - #[inline] - pub fn try_simple(&mut self, expr: &ConstExpr) -> Option<&Val> { - match expr.const_eval()? { - GlobalConstValue::I32(i) => Some(self.return_one(Val::I32(i))), - GlobalConstValue::I64(i) => Some(self.return_one(Val::I64(i))), - GlobalConstValue::F32(f) => Some(self.return_one(Val::F32(f))), - GlobalConstValue::F64(f) => Some(self.return_one(Val::F64(f))), - GlobalConstValue::V128(i) => Some(self.return_one(Val::V128(i.into()))), - } - } - - /// Evaluate the given const expression in the given context. - /// - /// Note that the `store` argument is an `OpaqueRootScope` which is used to - /// require that a GC rooting scope external to evaluation of this constant - /// is required. Constant expression evaluation may perform GC allocations - /// and itself trigger a GC meaning that all references must be rooted, - /// hence the external requirement of a rooting scope. - /// - /// # Panics - /// - /// This function will panic if `expr` is an invalid constant expression. - pub async fn eval( - &mut self, - store: &mut OpaqueRootScope<&mut StoreOpaque>, - limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - expr: &ConstExpr, - ) -> Result<&Val> { - // Same structure as `eval_int` above, except using `.await` and with a - // slightly different type signature here for this function. - if self.try_simple(expr).is_some() { - return Ok(&self.simple); - } - self.eval_loop(store, limiter, context, expr).await - } - - #[inline] - fn return_one(&mut self, val: Val) -> &Val { - self.simple = val; - &self.simple - } - - #[cold] - async fn eval_loop( - &mut self, - store: &mut OpaqueRootScope<&mut StoreOpaque>, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - expr: &ConstExpr, - ) -> Result<&Val> { - log::trace!("evaluating const expr: {expr:?}"); - - self.stack.clear(); - - // On GC-less builds ensure that this is always considered used an - // needed-mutable. - let _ = &mut limiter; - - for op in expr.ops() { - log::trace!("const-evaluating op: {op:?}"); - match op { - ConstOp::I32Const(i) => self.stack.push(Val::I32(*i))?, - ConstOp::I64Const(i) => self.stack.push(Val::I64(*i))?, - ConstOp::F32Const(f) => self.stack.push(Val::F32(*f))?, - ConstOp::F64Const(f) => self.stack.push(Val::F64(*f))?, - ConstOp::V128Const(v) => self.stack.push(Val::V128((*v).into()))?, - ConstOp::GlobalGet(g) => self.stack.push(context.global_get(store, *g)?)?, - ConstOp::RefNull(ty) => self.stack.push(Val::null_top(*ty))?, - ConstOp::RefFunc(f) => self.stack.push(context.ref_func(store, *f)?)?, - #[cfg(feature = "gc")] - ConstOp::RefI31 => { - let i = self.pop()?.unwrap_i32(); - let i31 = I31::wrapping_i32(i); - let r = AnyRef::_from_i31(&mut AutoAssertNoGc::new(store), i31); - self.stack.push(Val::AnyRef(Some(r)))?; - } - #[cfg(not(feature = "gc"))] - ConstOp::RefI31 => panic!("should not have validated"), - ConstOp::I32Add => { - let b = self.pop()?.unwrap_i32(); - let a = self.pop()?.unwrap_i32(); - self.stack.push(Val::I32(a.wrapping_add(b)))?; - } - ConstOp::I32Sub => { - let b = self.pop()?.unwrap_i32(); - let a = self.pop()?.unwrap_i32(); - self.stack.push(Val::I32(a.wrapping_sub(b)))?; - } - ConstOp::I32Mul => { - let b = self.pop()?.unwrap_i32(); - let a = self.pop()?.unwrap_i32(); - self.stack.push(Val::I32(a.wrapping_mul(b)))?; - } - ConstOp::I64Add => { - let b = self.pop()?.unwrap_i64(); - let a = self.pop()?.unwrap_i64(); - self.stack.push(Val::I64(a.wrapping_add(b)))?; - } - ConstOp::I64Sub => { - let b = self.pop()?.unwrap_i64(); - let a = self.pop()?.unwrap_i64(); - self.stack.push(Val::I64(a.wrapping_sub(b)))?; - } - ConstOp::I64Mul => { - let b = self.pop()?.unwrap_i64(); - let a = self.pop()?.unwrap_i64(); - self.stack.push(Val::I64(a.wrapping_mul(b)))?; - } - - #[cfg(not(feature = "gc"))] - ConstOp::StructNew { .. } - | ConstOp::StructNewDefault { .. } - | ConstOp::ArrayNew { .. } - | ConstOp::ArrayNewDefault { .. } - | ConstOp::ArrayNewFixed { .. } - | ConstOp::ExternConvertAny - | ConstOp::AnyConvertExtern => { - bail!( - "const expr evaluation error: struct operations are not \ - supported without the `gc` feature" - ) - } - - #[cfg(feature = "gc")] - ConstOp::StructNew { struct_type_index } => { - let interned_type_index = store.instance(context.instance).env_module().types - [*struct_type_index] - .unwrap_engine_type_index(); - let len = context.struct_fields_len(store, interned_type_index); - - if self.stack.len() < len { - bail!( - "const expr evaluation error: expected at least {len} values on the stack, found {}", - self.stack.len() - ) - } - - let start = self.stack.len() - len; - let s = context - .struct_new( - store, - limiter.as_deref_mut(), - interned_type_index, - &self.stack[start..], - ) - .await?; - self.stack.truncate(start); - self.stack.push(s)?; - } - - #[cfg(feature = "gc")] - ConstOp::StructNewDefault { struct_type_index } => { - let ty = store.instance(context.instance).env_module().types - [*struct_type_index] - .unwrap_engine_type_index(); - self.stack.push( - context - .struct_new_default(store, limiter.as_deref_mut(), ty) - .await?, - )?; - } - - #[cfg(feature = "gc")] - ConstOp::ArrayNew { array_type_index } => { - let ty = store.instance(context.instance).env_module().types[*array_type_index] - .unwrap_engine_type_index(); - let ty = ArrayType::from_shared_type_index(store.engine(), ty); - - let len = self.pop()?.unwrap_i32().cast_unsigned(); - - let elem = self.pop()?; - - let pre = ArrayRefPre::_new(store, ty); - let array = ArrayRef::_new_async( - store, - limiter.as_deref_mut(), - &pre, - &elem, - len, - context.asyncness, - ) - .await?; - - self.stack.push(Val::AnyRef(Some(array.into())))?; - } - - #[cfg(feature = "gc")] - ConstOp::ArrayNewDefault { array_type_index } => { - let ty = store.instance(context.instance).env_module().types[*array_type_index] - .unwrap_engine_type_index(); - let ty = ArrayType::from_shared_type_index(store.engine(), ty); - - let len = self.pop()?.unwrap_i32().cast_unsigned(); - - let elem = Val::default_for_ty(ty.element_type().unpack()) - .expect("type should have a default value"); - - let pre = ArrayRefPre::_new(store, ty); - let array = ArrayRef::_new_async( - store, - limiter.as_deref_mut(), - &pre, - &elem, - len, - context.asyncness, - ) - .await?; - - self.stack.push(Val::AnyRef(Some(array.into())))?; - } - - #[cfg(feature = "gc")] - ConstOp::ArrayNewFixed { - array_type_index, - array_size, - } => { - let ty = store.instance(context.instance).env_module().types[*array_type_index] - .unwrap_engine_type_index(); - let ty = ArrayType::from_shared_type_index(store.engine(), ty); - - let array_size = usize::try_from(*array_size).unwrap(); - if self.stack.len() < array_size { - bail!( - "const expr evaluation error: expected at least {array_size} values on the stack, found {}", - self.stack.len() - ) - } - - let start = self.stack.len() - array_size; - - let elems = self - .stack - .drain(start..) - .collect::>(); - - let pre = ArrayRefPre::_new(store, ty); - let array = ArrayRef::_new_fixed_async( - store, - limiter.as_deref_mut(), - &pre, - &elems, - context.asyncness, - ) - .await?; - - self.stack.push(Val::AnyRef(Some(array.into())))?; - } - - #[cfg(feature = "gc")] - ConstOp::ExternConvertAny => { - let mut store = AutoAssertNoGc::new(store); - let result = match self.pop()?.unwrap_anyref() { - Some(anyref) => Some(ExternRef::_convert_any(&mut store, *anyref)?), - None => None, - }; - self.stack.push(Val::ExternRef(result))?; - } - - #[cfg(feature = "gc")] - ConstOp::AnyConvertExtern => { - let mut store = AutoAssertNoGc::new(store); - let result = match self.pop()?.unwrap_externref() { - Some(externref) => Some(AnyRef::_convert_extern(&mut store, *externref)?), - None => None, - }; - self.stack.push(result.into())?; - } - } - } - - if self.stack.len() == 1 { - log::trace!("const expr evaluated to {:?}", self.stack[0]); - Ok(&self.stack[0]) - } else { - bail!( - "const expr evaluation error: expected 1 resulting value, found {}", - self.stack.len() - ) - } - } - - fn pop(&mut self) -> Result { - self.stack.pop().ok_or_else(|| { - format_err!( - "const expr evaluation error: attempted to pop from an empty \ - evaluation stack" - ) - }) - } -} - -#[cfg(test)] -mod tests { - use super::{ConstExprEvaluator, Val}; - use wasmtime_environ::{ConstExpr, ConstOp}; - - #[test] - fn pop_empty_stack() { - let mut evaluator = ConstExprEvaluator::default(); - let err = evaluator.pop().unwrap_err(); - assert!( - format!("{err}").contains("attempted to pop from an empty"), - "unexpected error: {err}" - ); - } - - #[test] - fn try_simple_scalar_types() { - let mut ev = ConstExprEvaluator::default(); - - let expr = ConstExpr::new([ConstOp::I32Const(7)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::I32(7)))); - - let expr = ConstExpr::new([ConstOp::I64Const(99)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::I64(99)))); - - let expr = ConstExpr::new([ConstOp::F32Const(0x3f800000)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::F32(_)))); - - let expr = ConstExpr::new([ConstOp::F64Const(0x3ff0000000000000)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::F64(_)))); - - let expr = ConstExpr::new([ConstOp::V128Const(0x12345678)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::V128(_)))); - } -} diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rs index 15dbbaae5db4..6f8129f47c6a 100644 --- a/crates/wasmtime/src/runtime/vm/cow.rs +++ b/crates/wasmtime/src/runtime/vm/cow.rs @@ -194,7 +194,7 @@ impl ModuleMemoryImages { // If there's no initialization for this memory known then we don't // need an image for the memory so push `None` and move on. - let init = match init { + let (offset, runtime_index) = match init { Some(init) => init, None => { memories.push(None)?; @@ -202,11 +202,14 @@ impl ModuleMemoryImages { } }; - let data_range = init.data.start as usize..init.data.end as usize; + let data_range = &module.runtime_data[*runtime_index]; + let data_range = usize::try_from(data_range.start).unwrap() + ..usize::try_from(data_range.end).unwrap(); + if module.memories[memory_index] .minimum_byte_size() .map_or(false, |mem_initial_len| { - init.offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len + *offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len }) { // The image is rounded up to multiples of the host OS page @@ -220,7 +223,7 @@ impl ModuleMemoryImages { return Ok(None); } - let offset_usize = match usize::try_from(init.offset) { + let offset_usize = match usize::try_from(*offset) { Ok(offset) => offset, Err(_) => return Ok(None), }; diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs index 38057577f108..61be8250cda1 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs @@ -1370,7 +1370,8 @@ mod tests { num_defined_globals: 0, num_defined_tags: 0, num_escaped_funcs: 0, - num_passive_data: 0, + num_runtime_data: 0, + has_startup_func: false, }); assert_eq!( diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs index 9b097c43e7d5..600d3c1a8820 100644 --- a/crates/wasmtime/src/runtime/vm/instance.rs +++ b/crates/wasmtime/src/runtime/vm/instance.rs @@ -2,7 +2,6 @@ //! wasm module (except its callstack and register state). An //! `InstanceHandle` is a reference-counting handle for an `Instance`. -use crate::Val; use crate::code::ModuleWithCode; use crate::module::ModuleRegistry; use crate::prelude::*; @@ -18,9 +17,7 @@ use crate::runtime::vm::{ GcStore, HostResult, Imports, ModuleRuntimeInfo, SendSyncPtr, VMGcRef, VMGlobalKind, VMStore, VMStoreRawPtr, VmPtr, VmSafe, WasmFault, catch_unwind_and_record_trap, }; -use crate::store::{ - AutoAssertNoGc, InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter, -}; +use crate::store::{InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter}; use crate::vm::{VMWasmCallFunction, ValRaw}; use alloc::sync::Arc; use core::alloc::Layout; @@ -35,10 +32,11 @@ use core::{mem, ptr}; use wasmtime_environ::ModuleInternedTypeIndex; use wasmtime_environ::error::OutOfMemory; use wasmtime_environ::{ - DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, EntityIndex, - EntityRef, FuncIndex, GlobalIndex, HostPtr, MemoryIndex, PassiveDataIndex, PassiveElemIndex, - PtrSize, TableIndex, TableInitialValue, TagIndex, VMCONTEXT_MAGIC, VMOffsets, - VMSharedTypeIndex, WasmRefType, packed_option::ReservedValue, + Abi, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, + DefinedTagIndex, EntityIndex, EntityRef, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex, + HostPtr, MemoryIndex, MemoryInitialization, ModuleStartup, PassiveElemIndex, PtrSize, + RuntimeDataIndex, TableIndex, TagIndex, VMCONTEXT_MAGIC, VMOffsets, VMSharedTypeIndex, + WasmRefType, packed_option::ReservedValue, }; #[cfg(feature = "wmemcheck")] use wasmtime_wmemcheck::Wmemcheck; @@ -170,7 +168,7 @@ impl Instance { ) -> Result { let module = req.runtime_info.env_module(); let memory_tys = &module.memories; - let passive_elements = TryVec::with_capacity(module.passive_elements.len())?; + let mut passive_elements = TryVec::with_capacity(module.passive_elements.len())?; #[cfg(feature = "wmemcheck")] let wmemcheck_state = if req.store.engine().config().wmemcheck { @@ -188,6 +186,11 @@ impl Instance { #[cfg(not(feature = "wmemcheck"))] let _ = memory_tys; + for (_, (ty, len)) in req.runtime_info.env_module().passive_elements.iter() { + let len = usize::try_from(*len).unwrap(); + passive_elements.push(PassiveElementSegment::new(*ty, len)?)?; + } + let mut ret = OwnedInstance::new(Instance { id: req.id, runtime_info: req.runtime_info.clone(), @@ -205,6 +208,7 @@ impl Instance { unsafe { ret.get_mut().initialize_vmctx(req.store, req.imports); } + Ok(ret) } @@ -447,6 +451,7 @@ impl Instance { } /// Get a locally defined or imported memory. + #[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))] pub(crate) fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition { if let Some(defined_index) = self.env_module().defined_memory_index(index) { self.memory(defined_index) @@ -594,6 +599,26 @@ impl Instance { unsafe { crate::Func::from_vm_func_ref(store, func_ref) } } + /// Returns a `Func` corresponding to the startup function for this + /// instance, if generated at compile time. + /// + /// # Safety + /// + /// The `store` parameter must be the store that owns this instance and the + /// functions that this instance can reference. + pub unsafe fn get_startup_func( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + store: StoreId, + ) -> Option { + let func_ref = self.get_start_func_ref(registry)?; + + // SAFETY: the validity of `func_ref` is guaranteed by the validity of + // `self`, and the contract that `store` must own `func_ref` is a + // contract of this function itself. + Some(unsafe { crate::Func::from_vm_func_ref(store, func_ref) }) + } + /// Lookup a table by index. /// /// # Panics @@ -820,11 +845,86 @@ impl Instance { return None; } - let Some(def_index) = self.env_module().defined_func_index(index) else { - debug_assert!(self.env_module().is_imported_function(index)); - return Some(self.imported_function(index).as_func_ref().into()); + match self.env_module().defined_func_index(index) { + Some(index) => self.initialize_defined_funcref(registry, index), + None => { + debug_assert!(self.env_module().is_imported_function(index)); + Some(self.imported_function(index).as_func_ref().into()) + } + } + } + + /// Initializes a defined function's `VMFuncRef` in-place and then returns a + /// pointer to that location. + fn initialize_defined_funcref( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + def_index: DefinedFuncIndex, + ) -> Option> { + let module = self.env_module(); + let index = module.func_index(def_index); + let func = &module.functions[index]; + let type_index = func.signature.unwrap_engine_type_index(); + let vmctx_offset = self.offsets().vmctx_func_ref(func.func_ref); + let array_to_wasm_key = FuncKey::ArrayToWasmTrampoline(module.module_index, def_index); + let wasm_key = FuncKey::DefinedWasmFunction(module.module_index, def_index); + // SAFETY: the type/offset/keys here are all valid for the defined + // function at `def_index`. + unsafe { + self.initialize_and_return_funcref( + registry, + type_index, + vmctx_offset, + array_to_wasm_key, + wasm_key, + ) + } + } + + fn get_start_func_ref( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + ) -> Option> { + let module = self.env_module(); + let type_index = match module.startup { + ModuleStartup::None => return None, + ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => { + t.unwrap_engine_type_index() + } }; + let vmctx_offset = self.offsets().vmctx_startup_func_ref(); + let array_to_wasm_key = FuncKey::ModuleStartup(Abi::Array, module.module_index); + let wasm_key = FuncKey::ModuleStartup(Abi::Wasm, module.module_index); + // SAFETY: the type/offset/keys here are all valid for the module + // startup function. + unsafe { + self.initialize_and_return_funcref( + registry, + type_index, + vmctx_offset, + array_to_wasm_key, + wasm_key, + ) + } + } + /// Common implementation of initializing a `VMFuncRef` stored within this + /// instance's `VMContext`. + /// + /// # Safety + /// + /// This function requires that `type_index` accurately describes this + /// function and `vmctx_offset` is indeed the correct offset for the + /// functions here. Effectively all the arguments here must be "logically + /// correct" for the `VMFuncRef` being initialized. + unsafe fn initialize_and_return_funcref( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + type_index: VMSharedTypeIndex, + vmctx_offset: u32, + array_to_wasm_key: FuncKey, + wasm_key: FuncKey, + ) -> Option> { // For now, we eagerly initialize an funcref struct in-place whenever // asked for a reference to it. This is mostly fine, because in practice // each funcref is unlikely to be requested more than a few times: @@ -846,9 +946,6 @@ impl Instance { // it's better for instantiation performance if we don't have to track // "is-initialized" state at all! - let func = &self.env_module().functions[index]; - let type_index = func.signature.unwrap_engine_type_index(); - let module_with_code = ModuleWithCode::in_store( registry, self.runtime_module() @@ -856,19 +953,13 @@ impl Instance { ) .expect("module not in store"); - let array_call = VmPtr::from( - NonNull::from( - module_with_code - .array_to_wasm_trampoline(def_index) - .expect("should have array-to-Wasm trampoline for escaping function"), - ) - .cast(), - ); + let array_call = + VmPtr::from(NonNull::from(module_with_code.function(array_to_wasm_key)).cast()); let wasm_call = Some(VmPtr::from( NonNull::new( module_with_code - .finished_function(def_index) + .function(wasm_key) .as_ptr() .cast::() .cast_mut(), @@ -880,9 +971,7 @@ impl Instance { // SAFETY: the offset calculated here should be correct with // `self.offsets` - let func_ref_ptr = unsafe { - self.vmctx_plus_offset_raw::(self.offsets().vmctx_func_ref(func.func_ref)) - }; + let func_ref_ptr = unsafe { self.vmctx_plus_offset_raw::(vmctx_offset) }; // SAFETY: the `func_ref_ptr` should be valid as it's within our // `VMContext` area. @@ -899,15 +988,16 @@ impl Instance { } /// Get the passive elements segment at the given index. - pub(crate) fn passive_element_segment(&self, passive: PassiveElemIndex) -> &[ValRaw] { - self.passive_elements[passive.index()].elements() + pub(crate) fn passive_element_segment( + self: Pin<&mut Self>, + passive: PassiveElemIndex, + ) -> &mut [ValRaw] { + self.passive_elements_mut()[passive.index()].elements_mut() } - pub(crate) fn passive_elements_mut( - self: Pin<&mut Self>, - ) -> Pin<&mut TryVec> { + pub(crate) fn passive_elements_mut(self: Pin<&mut Self>) -> &mut TryVec { // SAFETY: Not moving data out of `self`. - Pin::new(&mut unsafe { self.get_unchecked_mut() }.passive_elements) + &mut unsafe { self.get_unchecked_mut() }.passive_elements } /// Drop an element. @@ -946,7 +1036,7 @@ impl Instance { &self.runtime_info.wasm_data()[start..end] } - /// Returns the data for the passive segment identified by `index` + /// Returns the data for the runtime segment identified by `index` /// /// Does not take into account the dynamic size of the data pointed to by /// `index`, always returns the raw data from the module itself. @@ -954,8 +1044,8 @@ impl Instance { /// # Panics /// /// Panics if `index` is out-of-bounds. - fn passive_data(&self, index: PassiveDataIndex) -> &[u8] { - let range = self.env_module().passive_data[index].clone(); + fn runtime_data(&self, index: RuntimeDataIndex) -> &[u8] { + let range = self.env_module().runtime_data[index].clone(); self.wasm_data(range) } @@ -1013,10 +1103,7 @@ impl Instance { // that `i` may be outside the limits of the static // initialization so it's a fallible `get` instead of an index. let module = self.env_module(); - let precomputed = match &module.table_initialization.initial_values[idx] { - TableInitialValue::Null { precomputed } => precomputed, - TableInitialValue::Expr(_) => unreachable!(), - }; + let precomputed = &module.table_initialization[idx]; // Panicking here helps catch bugs rather than silently truncating by accident. let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned(); let func_ref = func_index @@ -1031,13 +1118,6 @@ impl Instance { self.get_defined_table(idx) } - /// Get a table by index regardless of whether it is locally-defined or an - /// imported, foreign table. - pub(crate) fn get_table(self: Pin<&mut Self>, table_index: TableIndex) -> &mut Table { - let (idx, instance) = self.defined_table_index_and_instance(table_index); - instance.get_defined_table(idx) - } - /// Get a locally-defined table. pub(crate) fn get_defined_table(self: Pin<&mut Self>, index: DefinedTableIndex) -> &mut Table { &mut self.tables_mut()[index].1 @@ -1237,9 +1317,21 @@ impl Instance { // after this, but if it's read then it'd hopefully crash faster than // leaving this undefined. unsafe { - for (index, _init) in module.global_initializers.iter() { + for i in 0..module.num_defined_globals() { + let index = DefinedGlobalIndex::new(i); instance.global_ptr(index).write(VMGlobalDefinition::new()); } + for (index, val) in module.global_initializers.iter() { + let mut def = VMGlobalDefinition::new(); + match val { + GlobalConstValue::I32(i) => *def.as_i32_mut() = *i, + GlobalConstValue::I64(i) => *def.as_i64_mut() = *i, + GlobalConstValue::F32(i) => *def.as_f32_bits_mut() = *i, + GlobalConstValue::F64(i) => *def.as_f64_bits_mut() = *i, + GlobalConstValue::V128(i) => def.set_u128(*i), + } + instance.global_ptr(*index).write(def); + } } // Initialize the defined tags @@ -1261,7 +1353,7 @@ impl Instance { } } - // Initialize the lengths of passive data segments. + // Initialize the lengths of runtime data segments. // // SAFETY: it's safe to initialize these lengths during initialization // here and the various types of pointers and such here should all be @@ -1269,17 +1361,48 @@ impl Instance { unsafe { let offsets = instance.runtime_info.offsets(); let mut lengths = - instance.vmctx_plus_offset_raw(offsets.vmctx_passive_data_lengths_begin()); + instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_lengths_begin()); let mut bases = - instance.vmctx_plus_offset_raw(offsets.vmctx_passive_data_bases_begin()); - for i in module.passive_data.keys() { - let data = instance.passive_data(i); + instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_bases_begin()); + for i in module.runtime_data.keys() { + let data = instance.runtime_data(i); lengths.write(u32::try_from(data.len()).unwrap()); lengths = lengths.add(1); bases.write(VmPtr::from(NonNull::from(data).cast::())); bases = bases.add(1); } } + + // This is the half of the strategy of implementing memory-init-cow data + // segments. Notably the compiled startup function, if present, will + // skip data segments that have a null pointer. Here each linear memory + // is tested to see if it needs initialization. If it does, then the + // data segment is left in-place (and the startup function will + // initialize linear memory). Otherwise the data segment is null'd out. + // If the startup function runs (e.g. something else in the module needs + // it), then the corresponding data segment's initialization will be + // skipped. + if let MemoryInitialization::Static { map } = &module.memory_initialization { + for (memory, init) in map { + let Some(memory) = module.defined_memory_index(memory) else { + continue; + }; + if instance.memories[memory].1.needs_init() { + continue; + } + if let Some((_offset, data)) = init { + let offsets = instance.runtime_info.offsets(); + unsafe { + instance + .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_length(*data)) + .write(0u32); + instance + .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_base(*data)) + .write(0usize); + } + } + } + } } /// Attempts to convert from the host `addr` specified to a WebAssembly @@ -1397,6 +1520,17 @@ impl Instance { // SAFETY: see `store_mut` above. unsafe { &mut self.get_unchecked_mut().wmemcheck_state } } + + pub(crate) fn needs_startup(&self) -> bool { + match self.env_module().startup { + ModuleStartup::None => false, + ModuleStartup::Always(_) => true, + ModuleStartup::IfMemoriesNeedInit(_) => self + .memories + .iter() + .any(|(_, (_, memory))| memory.needs_init()), + } + } } // SAFETY: `layout` should describe this accurately and `OwnedVMContext` is the @@ -1721,39 +1855,14 @@ pub(crate) struct PassiveElementSegment { impl PassiveElementSegment { /// Create a new passive element segment with the given capacity. pub(crate) fn new(ty: WasmRefType, capacity: usize) -> Result { + let mut elements = TryVec::with_capacity(capacity)?; + elements.resize_with(capacity, || ValRaw::null())?; Ok(Self { needs_gc_rooting: ty.is_vmgcref_type_and_not_i31(), - elements: TryVec::with_capacity(capacity)?, + elements, }) } - /// Push a value onto this passive element segment. - /// - /// NB: Does not type check the value, relies on callers to ensure the - /// value is of the correct type (generally, due to validation). - pub(crate) fn push(&mut self, store: &mut StoreOpaque, val: Val) -> Result<()> { - let mut val = { - let mut store = AutoAssertNoGc::new(store); - val.to_raw_(&mut store)? - }; - if self.needs_gc_rooting { - // Note that `anyref` accessors and constructors are used here - // without actually checking the type of this segment or value. The - // representation and handling of all three is the same which means - // that this should work out. - let gc_ref = val.get_anyref(); - debug_assert_eq!(gc_ref, val.get_exnref()); - debug_assert_eq!(gc_ref, val.get_externref()); - if let Some(gc_ref) = VMGcRef::from_raw_u32(gc_ref) { - if let Some(gc_store) = store.optional_gc_store_mut() { - val = ValRaw::anyref(gc_store.clone_gc_ref(&gc_ref).as_raw_u32()); - } - } - } - self.elements.push(val)?; - Ok(()) - } - /// Clear this segment's elements. pub(crate) fn clear(&mut self, mut gc_store: Option<&mut GcStore>) { let elements = mem::take(&mut self.elements); @@ -1776,12 +1885,6 @@ impl PassiveElementSegment { } /// The elements of this segment. - pub(crate) fn elements(&self) -> &[ValRaw] { - &self.elements - } - - /// The elements of this segment. - #[cfg(feature = "gc")] pub(crate) fn elements_mut(&mut self) -> &mut [ValRaw] { &mut self.elements } diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rs index e2f6ae082455..47e301ede32f 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs @@ -1,21 +1,16 @@ use crate::prelude::*; -use crate::runtime::vm::const_expr::{ConstEvalContext, ConstExprEvaluator}; use crate::runtime::vm::imports::Imports; use crate::runtime::vm::instance::{Instance, InstanceHandle}; use crate::runtime::vm::memory::Memory; use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::table::Table; use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo}; -use crate::store::{Asyncness, InstanceId, StoreOpaque, StoreResourceLimiter}; -use crate::vm::instance::PassiveElementSegment; -use crate::{OpaqueRootScope, Val}; +use crate::store::{InstanceId, StoreOpaque, StoreResourceLimiter}; use core::future::Future; +use core::mem; use core::pin::Pin; -use core::{mem, ptr}; use wasmtime_environ::{ - DefinedMemoryIndex, DefinedTableIndex, EntityRef, HostPtr, InitMemory, MemoryInitialization, - MemoryInitializer, MemoryKind, Module, SizeOverflow, TableInitialValue, TableSegmentElements, - Trap, VMOffsets, WasmRefType, + DefinedMemoryIndex, DefinedTableIndex, HostPtr, MemoryKind, Module, VMOffsets, }; #[cfg(feature = "gc")] @@ -499,445 +494,6 @@ impl dyn InstanceAllocator + '_ { } } -fn check_table_init_bounds( - store: &mut StoreOpaque, - instance: InstanceId, - module: &Module, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, -) -> Result<()> { - let mut store = OpaqueRootScope::new(store); - - for segment in module.table_initialization.segments.iter() { - let start = const_evaluator.eval_int(&mut store, context, &segment.offset)?; - let start = get_index(start, module.tables[segment.table_index].idx_type); - let end = start.checked_add(segment.elements.len()); - - let table = store.instance_mut(instance).get_table(segment.table_index); - match end { - Some(end) if end <= u64::try_from(table.size())? => { - // Initializer is in bounds - } - _ => { - bail!(Trap::TableOutOfBounds); - } - } - } - - Ok(()) -} - -async fn initialize_tables( - store: &mut StoreOpaque, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - let mut store = OpaqueRootScope::new(store); - for (table, init) in module.table_initialization.initial_values.iter() { - match init { - // Tables are always initially null-initialized at this time - TableInitialValue::Null { precomputed: _ } => {} - - TableInitialValue::Expr(expr) => { - let init = const_evaluator - .eval(&mut store, limiter.as_deref_mut(), context, expr) - .await?; - let idx = module.table_index(table); - let id = store.id(); - let table = store - .instance_mut(context.instance) - .get_exported_table(id, idx); - let size = table.size_(&store); - table._fill(&mut store, 0, init.ref_().unwrap(), size)?; - } - } - } - - // Note: if the module's table initializer state is in - // FuncTable mode, we will lazily initialize tables based on - // any statically-precomputed image of FuncIndexes, but there - // may still be "leftover segments" that could not be - // incorporated. So we have a unified handler here that - // iterates over all segments (Segments mode) or leftover - // segments (FuncTable mode) to initialize. - for segment in module.table_initialization.segments.iter() { - let start = const_evaluator.eval_int(&mut store, context, &segment.offset)?; - let start = get_index(start, module.tables[segment.table_index].idx_type); - - let end = start - .checked_add(segment.elements.len()) - .ok_or_else(|| Trap::TableOutOfBounds)?; - - let store_id = store.id(); - let table = { - let instance = store.instance(context.instance); - instance.get_exported_table(store_id, segment.table_index) - }; - - if end > table.size_(&store) { - return Err(Trap::TableOutOfBounds.into()); - } - - let positions = start..end; - - match &segment.elements { - TableSegmentElements::Functions(funcs) => { - for (i, func_idx) in positions.zip(funcs) { - let func = { - let (instance, registry) = - store.instance_and_module_registry_mut(context.instance); - // SAFETY: the `store_id` passed to `get_exported_func` is - // indeed the store that owns the function. - unsafe { instance.get_exported_func(registry, store_id, *func_idx) } - }; - table.set_(&mut store, i, func.into())?; - } - } - TableSegmentElements::Expressions { exprs, ty: _ } => { - for (i, expr) in positions.zip(exprs) { - let val = const_evaluator - .eval(&mut store, limiter.as_deref_mut(), context, expr) - .await?; - table.set_(&mut store, i, val.ref_().unwrap())?; - } - } - } - } - - Ok(()) -} - -fn get_index(val: &Val, ty: wasmtime_environ::IndexType) -> u64 { - match ty { - wasmtime_environ::IndexType::I32 => val.unwrap_i32().cast_unsigned().into(), - wasmtime_environ::IndexType::I64 => val.unwrap_i64().cast_unsigned(), - } -} - -fn get_memory_init_start( - store: &mut StoreOpaque, - init: &MemoryInitializer, - instance: InstanceId, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, -) -> Result { - let mut store = OpaqueRootScope::new(store); - const_evaluator - .eval_int(&mut store, context, &init.offset) - .map(|v| { - get_index( - v, - store.instance(instance).env_module().memories[init.memory_index].idx_type, - ) - }) -} - -fn check_memory_init_bounds( - store: &mut StoreOpaque, - instance: InstanceId, - initializers: &[MemoryInitializer], - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, -) -> Result<()> { - for init in initializers { - let memory = store.instance_mut(instance).get_memory(init.memory_index); - let start = get_memory_init_start(store, init, instance, context, const_evaluator)?; - let end = usize::try_from(start) - .ok() - .and_then(|start| start.checked_add(init.data.len())); - - match end { - Some(end) if end <= memory.current_length() => { - // Initializer is in bounds - } - _ => { - bail!(Trap::MemoryOutOfBounds); - } - } - } - - Ok(()) -} - -fn initialize_memories( - store: &mut StoreOpaque, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - // Delegates to the `init_memory` method which is sort of a duplicate of - // `instance.memory_init_segment` but is used at compile-time in other - // contexts so is shared here to have only one method of memory - // initialization. - // - // This call to `init_memory` notably implements all the bells and whistles - // so errors only happen if an out-of-bounds segment is found, in which case - // a trap is returned. - - struct InitMemoryAtInstantiation<'a> { - module: &'a Module, - store: &'a mut StoreOpaque, - context: &'a mut ConstEvalContext, - const_evaluator: &'a mut ConstExprEvaluator, - } - - impl InitMemory for InitMemoryAtInstantiation<'_> { - fn memory_size_in_bytes( - &mut self, - memory: wasmtime_environ::MemoryIndex, - ) -> Result { - let len = self - .store - .instance(self.context.instance) - .get_memory(memory) - .current_length(); - let len = u64::try_from(len).unwrap(); - Ok(len) - } - - fn eval_offset( - &mut self, - memory: wasmtime_environ::MemoryIndex, - expr: &wasmtime_environ::ConstExpr, - ) -> Option { - let mut store = OpaqueRootScope::new(&mut *self.store); - let val = self - .const_evaluator - .eval_int(&mut store, self.context, expr) - .ok()?; - Some(get_index( - val, - store.instance(self.context.instance).env_module().memories[memory].idx_type, - )) - } - - fn write( - &mut self, - memory_index: wasmtime_environ::MemoryIndex, - init: &wasmtime_environ::StaticMemoryInitializer, - ) -> bool { - // If this initializer applies to a defined memory but that memory - // doesn't need initialization, due to something like copy-on-write - // pre-initializing it via mmap magic, then this initializer can be - // skipped entirely. - let instance = self.store.instance_mut(self.context.instance); - if let Some(memory_index) = self.module.defined_memory_index(memory_index) { - if !instance.memories[memory_index].1.needs_init() { - return true; - } - } - let memory = instance.get_memory(memory_index); - - unsafe { - let src = instance.wasm_data(init.data.clone()); - let offset = usize::try_from(init.offset).unwrap(); - let dst = memory.base.as_ptr().add(offset); - - assert!(offset + src.len() <= memory.current_length()); - - // FIXME audit whether this is safe in the presence of shared - // memory - // (https://github.com/bytecodealliance/wasmtime/issues/4203). - ptr::copy_nonoverlapping(src.as_ptr(), dst, src.len()) - } - true - } - } - - let ok = module - .memory_initialization - .init_memory(&mut InitMemoryAtInstantiation { - module, - store, - context, - const_evaluator, - }); - if !ok { - return Err(Trap::MemoryOutOfBounds.into()); - } - - Ok(()) -} - -fn check_init_bounds( - store: &mut StoreOpaque, - instance: InstanceId, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - check_table_init_bounds(store, instance, module, context, const_evaluator)?; - - match &module.memory_initialization { - MemoryInitialization::Segmented(initializers) => { - check_memory_init_bounds(store, instance, initializers, context, const_evaluator)?; - } - // Statically validated already to have everything in-bounds. - MemoryInitialization::Static { .. } => {} - } - - Ok(()) -} - -async fn initialize_globals( - store: &mut StoreOpaque, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - assert!(core::ptr::eq( - &**store.instance(context.instance).env_module(), - module - )); - - let mut store = OpaqueRootScope::new(store); - - for (index, init) in module.global_initializers.iter() { - // Attempt a simple, synchronous evaluation before hitting the - // general-purpose `.await` point below. This benchmarks ~15% faster in - // instantiation vs just falling through to `.await` below. - let val = if let Some(val) = const_evaluator.try_simple(init) { - val - } else { - const_evaluator - .eval(&mut store, limiter.as_deref_mut(), context, init) - .await? - }; - - let id = store.id(); - let index = module.global_index(index); - let mut instance = store.instance_mut(context.instance); - - #[cfg(feature = "wmemcheck")] - if index.as_u32() == 0 - && module.globals[index].wasm_ty == wasmtime_environ::WasmValType::I32 - { - if let Some(wmemcheck) = instance.as_mut().wmemcheck_state_mut() { - let size = usize::try_from(val.unwrap_i32()).unwrap(); - wmemcheck.set_stack_size(size); - } - } - - let global = instance.as_mut().get_exported_global(id, index); - - // Note that mutability is bypassed here because this is, by definition, - // initialization of globals meaning that if it's an immutable global - // this is the one and only write. - // - // SAFETY: this is a valid module so `val` should have the correct type - // for this global, and it's safe to write to a global for the first - // time as-is happening here. - unsafe { - global.set_unchecked(&mut store, &val)?; - } - } - Ok(()) -} - -async fn initialize_passive_elements( - store: &mut StoreOpaque, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - let store_id = store.id(); - - let instance = store.instance_mut(context.instance); - debug_assert!(instance.passive_elements.is_empty()); - instance - .passive_elements_mut() - .reserve(module.passive_elements.len())?; - - for (idx, segment) in &module.passive_elements { - match segment { - TableSegmentElements::Functions(func_indices) => { - let mut segment = - PassiveElementSegment::new(WasmRefType::FUNCREF, func_indices.len())?; - for func_idx in func_indices { - let (instance, registry) = - store.instance_and_module_registry_mut(context.instance); - // SAFETY: `store_id` is for the store that owns this instance. - let func = unsafe { instance.get_exported_func(registry, store_id, *func_idx) }; - segment.push(store, func.into())?; - } - let instance = store.instance_mut(context.instance); - debug_assert_eq!(instance.passive_elements.len(), idx.index()); - instance.passive_elements_mut().push(segment)?; - } - TableSegmentElements::Expressions { ty, exprs } => { - let mut segment = PassiveElementSegment::new(*ty, exprs.len())?; - for expr in exprs { - let mut store = OpaqueRootScope::new(&mut *store); - let val = const_evaluator - .eval(&mut store, limiter.as_deref_mut(), context, expr) - .await?; - segment.push(&mut store, *val)?; - } - let instance = store.instance_mut(context.instance); - debug_assert_eq!(instance.passive_elements.len(), idx.index()); - instance.passive_elements_mut().push(segment)?; - } - } - } - - debug_assert_eq!( - module.passive_elements.len(), - store.instance(context.instance).passive_elements.len() - ); - Ok(()) -} - -pub async fn initialize_instance( - store: &mut StoreOpaque, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - instance: InstanceId, - module: &Module, - is_bulk_memory: bool, - asyncness: Asyncness, -) -> Result<()> { - let mut context = ConstEvalContext::new(instance, asyncness); - let mut const_evaluator = ConstExprEvaluator::default(); - - // If bulk memory is not enabled, bounds check the data and element segments before - // making any changes. With bulk memory enabled, initializers are processed - // in-order and side effects are observed up to the point of an out-of-bounds - // initializer, so the early checking is not desired. - if !is_bulk_memory { - check_init_bounds(store, instance, &mut context, &mut const_evaluator, module)?; - } - - initialize_globals( - store, - limiter.as_deref_mut(), - &mut context, - &mut const_evaluator, - module, - ) - .await?; - - initialize_tables( - store, - limiter.as_deref_mut(), - &mut context, - &mut const_evaluator, - module, - ) - .await?; - - initialize_memories(store, &mut context, &mut const_evaluator, &module)?; - - if is_bulk_memory { - initialize_passive_elements(store, limiter, &mut context, &mut const_evaluator, &module) - .await?; - } - - Ok(()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 241057fa7128..f8641c87d6df 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -341,8 +341,7 @@ fn passive_elem_segment_base( store .instance_mut(instance) .passive_element_segment(elem_index) - .as_ptr() - .cast_mut() + .as_mut_ptr() .cast() } diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index ab097b978eb6..83844fb4eeff 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -4,7 +4,6 @@ mod vm_host_func_context; pub use self::vm_host_func_context::VMArrayCallHostFuncContext; -use crate::bail_bug; use crate::prelude::*; use crate::runtime::vm::{InterpreterRef, VMGcRef, VmPtr, VmSafe, f32x4, f64x2, i8x16}; use crate::store::StoreOpaque; @@ -20,7 +19,6 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use wasmtime_environ::{ BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, NUM_COMPONENT_CONTEXT_SLOTS, VMCONTEXT_MAGIC, VMSharedTypeIndex, - WasmHeapTopType, WasmValType, }; /// A function pointer that exposes the array calling convention. @@ -600,86 +598,6 @@ impl VMGlobalDefinition { Self { storage: [0; 16] } } - /// Create a `VMGlobalDefinition` from a `ValRaw`. - /// - /// # Unsafety - /// - /// This raw value's type must match the given `WasmValType`. - pub unsafe fn from_val_raw( - store: &mut StoreOpaque, - wasm_ty: WasmValType, - raw: ValRaw, - ) -> Result { - let mut global = Self::new(); - unsafe { - match wasm_ty { - WasmValType::I32 => *global.as_i32_mut() = raw.get_i32(), - WasmValType::I64 => *global.as_i64_mut() = raw.get_i64(), - WasmValType::F32 => *global.as_f32_bits_mut() = raw.get_f32(), - WasmValType::F64 => *global.as_f64_bits_mut() = raw.get_f64(), - WasmValType::V128 => global.set_u128(raw.get_v128()), - WasmValType::Ref(r) => match r.heap_type.top() { - WasmHeapTopType::Extern => { - let r = VMGcRef::from_raw_u32(raw.get_externref()); - global.init_gc_ref(store, r.as_ref())? - } - WasmHeapTopType::Any => { - let r = VMGcRef::from_raw_u32(raw.get_anyref()); - global.init_gc_ref(store, r.as_ref())? - } - WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(), - WasmHeapTopType::Cont => *global.as_func_ref_mut() = raw.get_funcref().cast(), // TODO(#10248): temporary hack. - WasmHeapTopType::Exn => { - let r = VMGcRef::from_raw_u32(raw.get_exnref()); - global.init_gc_ref(store, r.as_ref())? - } - }, - } - } - Ok(global) - } - - /// Get this global's value as a `ValRaw`. - /// - /// # Unsafety - /// - /// This global's value's type must match the given `WasmValType`. - pub unsafe fn to_val_raw( - &self, - store: &mut StoreOpaque, - wasm_ty: WasmValType, - ) -> Result { - unsafe { - Ok(match wasm_ty { - WasmValType::I32 => ValRaw::i32(*self.as_i32()), - WasmValType::I64 => ValRaw::i64(*self.as_i64()), - WasmValType::F32 => ValRaw::f32(*self.as_f32_bits()), - WasmValType::F64 => ValRaw::f64(*self.as_f64_bits()), - WasmValType::V128 => ValRaw::v128(self.get_u128()), - WasmValType::Ref(r) => match r.heap_type.top() { - WasmHeapTopType::Extern => ValRaw::externref(match self.as_gc_ref() { - Some(r) => store.clone_gc_ref(r).as_raw_u32(), - None => 0, - }), - WasmHeapTopType::Any => ValRaw::anyref({ - match self.as_gc_ref() { - Some(r) => store.clone_gc_ref(r).as_raw_u32(), - None => 0, - } - }), - WasmHeapTopType::Exn => ValRaw::exnref({ - match self.as_gc_ref() { - Some(r) => store.clone_gc_ref(r).as_raw_u32(), - None => 0, - } - }), - WasmHeapTopType::Func => ValRaw::funcref(self.as_func_ref().cast()), - WasmHeapTopType::Cont => bail_bug!("unimplemented"), // FIXME: #10248 stack switching support. - }, - }) - } - } - /// Return a reference to the value as an i32. pub unsafe fn as_i32(&self) -> &i32 { unsafe { &*(self.storage.as_ref().as_ptr().cast::()) } diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rs index 3939c26ff777..6a9a7f29828c 100644 --- a/crates/winch/src/compiler.rs +++ b/crates/winch/src/compiler.rs @@ -167,25 +167,15 @@ impl wasmtime_environ::Compiler for Compiler { }) } - fn compile_array_to_wasm_trampoline( - &self, - translation: &ModuleTranslation<'_>, - types: &ModuleTypesBuilder, - key: FuncKey, - symbol: &str, - ) -> Result { - self.trampolines - .compile_array_to_wasm_trampoline(translation, types, key, symbol) - } - - fn compile_wasm_to_array_trampoline( + fn compile_trampoline( &self, - wasm_func_ty: &wasmtime_environ::WasmFuncType, + translation: Option<&ModuleTranslation<'_>>, key: FuncKey, + types: &ModuleTypesBuilder, symbol: &str, ) -> Result { self.trampolines - .compile_wasm_to_array_trampoline(wasm_func_ty, key, symbol) + .compile_trampoline(translation, key, types, symbol) } fn append_code( @@ -236,14 +226,6 @@ impl wasmtime_environ::Compiler for Compiler { self.isa.create_systemv_cie() } - fn compile_wasm_to_builtin( - &self, - key: FuncKey, - symbol: &str, - ) -> Result { - self.trampolines.compile_wasm_to_builtin(key, symbol) - } - fn compiled_function_relocation_targets<'a>( &'a self, func: &'a dyn Any, @@ -280,45 +262,14 @@ impl wasmtime_environ::Compiler for NoInlineCompiler { Ok(body) } - fn compile_array_to_wasm_trampoline( - &self, - translation: &ModuleTranslation<'_>, - types: &ModuleTypesBuilder, - key: FuncKey, - symbol: &str, - ) -> Result { - let mut body = self - .0 - .compile_array_to_wasm_trampoline(translation, types, key, symbol)?; - if let Some(c) = self.0.inlining_compiler() { - c.finish_compiling(&mut body, None, symbol) - .map_err(|e| CompileError::Codegen(e.to_string()))?; - } - Ok(body) - } - - fn compile_wasm_to_array_trampoline( - &self, - wasm_func_ty: &wasmtime_environ::WasmFuncType, - key: FuncKey, - symbol: &str, - ) -> Result { - let mut body = self - .0 - .compile_wasm_to_array_trampoline(wasm_func_ty, key, symbol)?; - if let Some(c) = self.0.inlining_compiler() { - c.finish_compiling(&mut body, None, symbol) - .map_err(|e| CompileError::Codegen(e.to_string()))?; - } - Ok(body) - } - - fn compile_wasm_to_builtin( + fn compile_trampoline( &self, + translation: Option<&ModuleTranslation<'_>>, key: FuncKey, + types: &ModuleTypesBuilder, symbol: &str, ) -> Result { - let mut body = self.0.compile_wasm_to_builtin(key, symbol)?; + let mut body = self.0.compile_trampoline(translation, key, types, symbol)?; if let Some(c) = self.0.inlining_compiler() { c.finish_compiling(&mut body, None, symbol) .map_err(|e| CompileError::Codegen(e.to_string()))?; @@ -381,7 +332,7 @@ impl wasmtime_environ::Compiler for NoInlineCompiler { #[cfg(feature = "component-model")] impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler { - fn compile_trampoline( + fn compile_component_trampoline( &self, component: &wasmtime_environ::component::ComponentTranslation, types: &wasmtime_environ::component::ComponentTypesBuilder, @@ -393,7 +344,7 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler { let mut body = self .0 .component_compiler() - .compile_trampoline(component, types, key, abi, tunables, symbol)?; + .compile_component_trampoline(component, types, key, abi, tunables, symbol)?; if let Some(c) = self.0.inlining_compiler() { c.finish_compiling(&mut body, None, symbol) .map_err(|e| CompileError::Codegen(e.to_string()))?; diff --git a/src/commands/objdump.rs b/src/commands/objdump.rs index a81dc2a8be5f..c85607b63459 100644 --- a/src/commands/objdump.rs +++ b/src/commands/objdump.rs @@ -205,6 +205,7 @@ impl ObjdumpCommand { || name.ends_with("_array_call") || name.ends_with("_wasm_call") || name.contains("unsafe-intrinsics-") + || name.contains("module_start") { Func::Trampoline } else if name.contains("libcall") || name.starts_with("component") { diff --git a/tests/all/arrays.rs b/tests/all/arrays.rs index 31f68b306e9a..b42a38855dd8 100644 --- a/tests/all/arrays.rs +++ b/tests/all/arrays.rs @@ -1002,6 +1002,7 @@ fn issue_13034_array_layout_overflow() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn host_arrayref_has_trace_info_for_gc() -> Result<()> { for collector in [Collector::Copying, Collector::DeferredReferenceCounting] { println!("Using GC collector: {collector:?}"); diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs index 96d45dcebd0f..a7f34fb7fcf8 100644 --- a/tests/all/cli_tests.rs +++ b/tests/all/cli_tests.rs @@ -3051,11 +3051,11 @@ fn profile_guest() -> Result<()> { None, )?; - assert!(output.status.success()); let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); println!("> stdout:\n{stdout}"); println!("> stderr:\n{stderr}"); + assert!(output.status.success()); assert!(!stderr.contains("Error")); let out_json = std::fs::read_to_string(format!("{dir}/out.json")).unwrap(); println!("> out.json:\n{out_json}"); diff --git a/tests/all/exceptions.rs b/tests/all/exceptions.rs index e4f326877a8c..cde1dec7424e 100644 --- a/tests/all/exceptions.rs +++ b/tests/all/exceptions.rs @@ -216,6 +216,7 @@ fn exception_across_no_wasm(config: &mut Config) -> Result<()> { } #[wasmtime_test(wasm_features(gc, exceptions))] +#[cfg_attr(miri, ignore)] fn gc_with_exnref_global(config: &mut Config) -> Result<()> { let engine = Engine::new(config)?; let mut store = Store::new(&engine, ()); diff --git a/tests/all/exnrefs.rs b/tests/all/exnrefs.rs index 6251d01ff669..de0b73efbba3 100644 --- a/tests/all/exnrefs.rs +++ b/tests/all/exnrefs.rs @@ -85,6 +85,7 @@ fn exn_objects() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn host_exnref_has_trace_info_for_gc() -> Result<()> { for collector in [Collector::Copying, Collector::DeferredReferenceCounting] { println!("Using GC collector: {collector:?}"); diff --git a/tests/all/fuel.wast b/tests/all/fuel.wast index 46613616661f..f918eb94aca3 100644 --- a/tests/all/fuel.wast +++ b/tests/all/fuel.wast @@ -1,11 +1,11 @@ (assert_fuel 0 (module)) -(assert_fuel 1 +(assert_fuel 3 (module (func $f) (start $f))) -(assert_fuel 2 +(assert_fuel 4 (module (func $f i32.const 0 @@ -13,7 +13,7 @@ ) (start $f))) -(assert_fuel 1 +(assert_fuel 3 (module (func $f block @@ -21,14 +21,14 @@ ) (start $f))) -(assert_fuel 1 +(assert_fuel 3 (module (func $f unreachable ) (start $f))) -(assert_fuel 7 +(assert_fuel 9 (module (func $f i32.const 0 @@ -41,7 +41,7 @@ ) (start $f))) -(assert_fuel 1 +(assert_fuel 3 (module (func $f return @@ -55,7 +55,7 @@ ) (start $f))) -(assert_fuel 3 +(assert_fuel 5 (module (func $f i32.const 0 @@ -65,7 +65,7 @@ ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f i32.const 1 @@ -76,7 +76,7 @@ ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f i32.const 1 @@ -89,7 +89,7 @@ ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f i32.const 0 @@ -102,7 +102,7 @@ ) (start $f))) -(assert_fuel 3 +(assert_fuel 5 (module (func $f block @@ -114,7 +114,7 @@ ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f block @@ -127,7 +127,7 @@ (start $f))) ;; count code before unreachable -(assert_fuel 2 +(assert_fuel 4 (module (func $f i32.const 0 @@ -136,7 +136,7 @@ (start $f))) ;; count code before return -(assert_fuel 2 +(assert_fuel 4 (module (func $f i32.const 0 @@ -145,14 +145,14 @@ (start $f))) ;; cross-function fuel works -(assert_fuel 3 +(assert_fuel 5 (module (func $f call $other ) (func $other) (start $f))) -(assert_fuel 5 +(assert_fuel 7 (module (func $f i32.const 0 @@ -162,7 +162,7 @@ ) (func $other (param i32)) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f call $other @@ -172,7 +172,7 @@ i32.const 0 ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f i32.const 0 @@ -183,14 +183,14 @@ (start $f))) ;; loops! -(assert_fuel 1 +(assert_fuel 3 (module (func $f loop end ) (start $f))) -(assert_fuel 53 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func +(assert_fuel 55 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func (module (func $f (local i32) @@ -207,7 +207,7 @@ ) (start $f))) -(assert_fuel 105 +(assert_fuel 107 (module (memory 1) (func $f @@ -218,7 +218,7 @@ ) (start $f))) -(assert_fuel 105 +(assert_fuel 107 (module (memory 1) (func $f @@ -229,7 +229,7 @@ ) (start $f))) -(assert_fuel 25 +(assert_fuel 27 (module (memory 1) (func $f @@ -241,7 +241,7 @@ (start $f) (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))) -(assert_fuel 105 +(assert_fuel 107 (module (table 100 funcref) (func $f @@ -252,7 +252,7 @@ ) (start $f))) -(assert_fuel 105 +(assert_fuel 107 (module (table 100 funcref) (func $f @@ -263,7 +263,7 @@ ) (start $f))) -(assert_fuel 104 +(assert_fuel 106 (module (table 0 funcref) (func $f @@ -274,7 +274,7 @@ ) (start $f))) -(assert_fuel 25 +(assert_fuel 27 (module (table 20 funcref) (func $f @@ -286,7 +286,7 @@ (start $f) (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f))) -(assert_fuel 107 +(assert_fuel 211 (module (type $a (array (mut i8))) (global $a (ref $a) (array.new_default $a (i32.const 100))) @@ -300,7 +300,7 @@ ) (start $f))) -(assert_fuel 106 +(assert_fuel 210 (module (type $a (array (mut i8))) (global $a (ref $a) (array.new_default $a (i32.const 100))) @@ -313,7 +313,7 @@ ) (start $f))) -(assert_fuel 24 +(assert_fuel 26 (module (type $a (array (mut i8))) (func $f @@ -325,7 +325,7 @@ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") (start $f))) -(assert_fuel 26 +(assert_fuel 130 (module (type $a (array (mut i8))) (global $a (ref $a) (array.new_default $a (i32.const 100))) @@ -339,7 +339,7 @@ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") (start $f))) -(assert_fuel 24 +(assert_fuel 26 (module (type $a (array (mut funcref))) (func $f @@ -351,7 +351,7 @@ (start $f) (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f))) -(assert_fuel 26 +(assert_fuel 130 (module (type $a (array (mut funcref))) (global $a (ref $a) (array.new_default $a (i32.const 100))) @@ -365,7 +365,7 @@ (start $f) (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f))) -(assert_fuel 103 +(assert_fuel 105 (module (type $a (array (mut funcref))) (func $f @@ -375,7 +375,7 @@ ) (start $f))) -(assert_fuel 104 +(assert_fuel 106 (module (type $a (array (mut funcref))) (func $f diff --git a/tests/all/gc.rs b/tests/all/gc.rs index 7301fe6bf2c8..b86a73f0b4ae 100644 --- a/tests/all/gc.rs +++ b/tests/all/gc.rs @@ -373,6 +373,7 @@ fn table_drops_externref() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn global_init_no_leak() -> Result<()> { let (mut store, module) = ref_types_module( false, @@ -1157,6 +1158,7 @@ fn issue_9669() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn drc_transitive_drop_cons_list() -> Result<()> { let _ = env_logger::try_init(); @@ -3315,6 +3317,8 @@ fn typed_option_noextern() -> Result<()> { /// A test that performs a GC without actually compiling or running any Wasm /// functions so we can run this test under MIRI. #[test] +// FIXME: need to fold this into other miri testing which precompiles +#[cfg_attr(miri, ignore)] fn miri_gc_smoke_test() -> Result<()> { for collector in [ Collector::Copying, diff --git a/tests/all/memory.rs b/tests/all/memory.rs index 8897ad081d70..f321f5d98c36 100644 --- a/tests/all/memory.rs +++ b/tests/all/memory.rs @@ -510,6 +510,7 @@ fn dynamic_extra_growth_unchanged_pointer(config: &mut Config) -> Result<()> { // determining this failure we shouldn't hit any overflows or anything like that // (checked via debug-mode tests). #[wasmtime_test] +#[cfg_attr(miri, ignore)] fn memory64_maximum_minimum(config: &mut Config) -> Result<()> { config.wasm_memory64(true); let engine = Engine::new(&config)?; diff --git a/tests/all/module.rs b/tests/all/module.rs index da99f7442b70..6a4daff1a78b 100644 --- a/tests/all/module.rs +++ b/tests/all/module.rs @@ -141,6 +141,7 @@ fn serialize_deterministic() { // into an initialization image doesn't unnecessarily create a massive module by // accident with a very large initialization image in it. #[test] +#[cfg_attr(miri, ignore)] fn serialize_not_overly_massive() -> Result<()> { let mut config = Config::new(); config.memory_guaranteed_dense_image_size(1 << 20); diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs index c14a6dbbc587..ffb7482003ca 100644 --- a/tests/all/pooling_allocator.rs +++ b/tests/all/pooling_allocator.rs @@ -98,6 +98,7 @@ fn memory_limit() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn memory_init() -> Result<()> { let mut pool = crate::small_pool_config(); pool.max_memory_size(2 << 16).table_elements(0); @@ -441,6 +442,7 @@ fn total_core_instances_limit() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn preserve_data_segments() -> Result<()> { let mut pool = crate::small_pool_config(); pool.total_memories(2); @@ -516,6 +518,7 @@ fn multi_memory_with_imported_memories() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn drop_externref_global_during_module_init() -> Result<()> { struct Limiter; @@ -1248,6 +1251,7 @@ fn decommit_batching() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn tricky_empty_table_with_empty_virtual_memory_alloc() -> Result<()> { // Configure the pooling allocator to have no access to virtual memory, e.g. // no table elements but a single table. This should technically support a @@ -1458,6 +1462,7 @@ fn pooling_reuse_resets() -> Result<()> { // asserts that the previous image is indeed not available any more as that // would otherwise mean data was leaked between modules. #[test] +#[cfg_attr(miri, ignore)] fn memory_reset_if_instantiation_fails() -> Result<()> { struct Limiter; diff --git a/tests/all/structs.rs b/tests/all/structs.rs index 59e14ae087db..652f028f31d0 100644 --- a/tests/all/structs.rs +++ b/tests/all/structs.rs @@ -912,6 +912,7 @@ fn issue_9714(config: &mut Config) -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn host_structref_has_trace_info_for_gc() -> Result<()> { for collector in [Collector::Copying, Collector::DeferredReferenceCounting] { println!("Using GC collector: {collector:?}"); diff --git a/tests/all/table.rs b/tests/all/table.rs index b4375c479a46..6d8d9bd843ff 100644 --- a/tests/all/table.rs +++ b/tests/all/table.rs @@ -368,6 +368,7 @@ fn host_table_keep_type_registration() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn gc_store_with_table_initializers() -> Result<()> { let mut config = Config::new(); config.wasm_gc(true); diff --git a/tests/disas/gc/array-init-data.wat b/tests/disas/gc/array-init-data.wat index 39ff89c7984d..c44bdab873d9 100644 --- a/tests/disas/gc/array-init-data.wat +++ b/tests/disas/gc/array-init-data.wat @@ -27,8 +27,8 @@ ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32): ;; @002a trapz v2, user16 -;; @002a v57 = load.i64 notrap aligned readonly can_move v0+8 -;; @002a v7 = load.i64 notrap aligned readonly can_move v57+32 +;; @002a v58 = load.i64 notrap aligned readonly can_move v0+8 +;; @002a v7 = load.i64 notrap aligned readonly can_move v58+32 ;; @002a v6 = uextend.i64 v2 ;; @002a v8 = iadd v7, v6 ;; @002a v9 = iconst.i64 16 @@ -40,22 +40,23 @@ ;; @002a v12 = uextend.i64 v11 ;; @002a v17 = icmp ugt v16, v12 ;; @002a trapnz v17, user17 -;; @002a v26 = uload32 notrap aligned v0+56 -;; @002a v27 = uextend.i64 v4 -;; @002a v30 = iadd v27, v14 -;; @002a v31 = icmp ugt v30, v26 -;; @002a trapnz v31, heap_oob -;; @002a v33 = load.i64 notrap aligned v0+48 -;; @002a v40 = load.i64 notrap aligned v57+40 -;; v53 = iconst.i64 20 -;; @002a v21 = iadd v8, v53 ; v53 = 20 +;; @002a v26 = load.i32 notrap aligned v0+56 +;; @002a v28 = uextend.i64 v4 +;; @002a v31 = iadd v28, v14 +;; @002a v27 = uextend.i64 v26 +;; @002a v32 = icmp ugt v31, v27 +;; @002a trapnz v32, heap_oob +;; @002a v34 = load.i64 notrap aligned v0+48 +;; @002a v41 = load.i64 notrap aligned v58+40 +;; v54 = iconst.i64 20 +;; @002a v21 = iadd v8, v54 ; v54 = 20 ;; @002a v24 = iadd v21, v13 -;; @002a v42 = uadd_overflow_trap v24, v14, user2 -;; @002a v41 = iadd v7, v40 -;; @002a v43 = icmp ugt v42, v41 -;; @002a trapnz v43, user2 -;; @002a v35 = iadd v33, v27 -;; @002a call fn0(v0, v24, v35, v14) +;; @002a v43 = uadd_overflow_trap v24, v14, user2 +;; @002a v42 = iadd v7, v41 +;; @002a v44 = icmp ugt v43, v42 +;; @002a trapnz v44, user2 +;; @002a v36 = iadd v34, v28 +;; @002a call fn0(v0, v24, v36, v14) ;; @002e jump block1 ;; ;; block1: diff --git a/tests/disas/gc/array-new-data.wat b/tests/disas/gc/array-new-data.wat index 2ff85b8dfbfb..29e109b03e99 100644 --- a/tests/disas/gc/array-new-data.wat +++ b/tests/disas/gc/array-new-data.wat @@ -92,96 +92,98 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @0025 v6 = uload32 notrap aligned v0+56 -;; @0025 v7 = uextend.i64 v2 -;; @0025 v8 = uextend.i64 v3 -;; @0025 v10 = iadd v7, v8 -;; @0025 v11 = icmp ugt v10, v6 -;; @0025 trapnz v11, heap_oob -;; @0025 v13 = load.i64 notrap aligned v0+48 -;; v124 = iconst.i64 32 -;; @0025 v19 = ushr v8, v124 ; v124 = 32 -;; @0025 trapnz v19, user18 -;; @0025 v16 = iconst.i32 20 -;; @0025 v21 = uadd_overflow_trap v16, v3, user18 ; v16 = 20 -;; @0025 v23 = load.i64 notrap aligned readonly can_move v0+32 -;; @0025 v24 = load.i32 notrap aligned v23 -;; @0025 v25 = load.i32 notrap aligned v23+4 -;; @0025 v31 = uextend.i64 v24 -;; @0025 v26 = uextend.i64 v21 -;; @0025 v27 = iconst.i64 15 -;; @0025 v29 = iadd v26, v27 ; v27 = 15 -;; @0025 v28 = iconst.i64 -16 -;; @0025 v30 = band v29, v28 ; v28 = -16 -;; @0025 v32 = iadd v31, v30 -;; @0025 v33 = uextend.i64 v25 -;; @0025 v34 = icmp ule v32, v33 -;; @0025 brif v34, block2, block3 +;; @0025 v6 = load.i32 notrap aligned v0+56 +;; @0025 v8 = uextend.i64 v2 +;; @0025 v9 = uextend.i64 v3 +;; @0025 v11 = iadd v8, v9 +;; @0025 v7 = uextend.i64 v6 +;; @0025 v12 = icmp ugt v11, v7 +;; @0025 trapnz v12, heap_oob +;; @0025 v14 = load.i64 notrap aligned v0+48 +;; v126 = iconst.i64 32 +;; @0025 v20 = ushr v9, v126 ; v126 = 32 +;; @0025 trapnz v20, user18 +;; @0025 v17 = iconst.i32 20 +;; @0025 v22 = uadd_overflow_trap v17, v3, user18 ; v17 = 20 +;; @0025 v24 = load.i64 notrap aligned readonly can_move v0+32 +;; @0025 v25 = load.i32 notrap aligned v24 +;; @0025 v26 = load.i32 notrap aligned v24+4 +;; @0025 v32 = uextend.i64 v25 +;; @0025 v27 = uextend.i64 v22 +;; @0025 v28 = iconst.i64 15 +;; @0025 v30 = iadd v27, v28 ; v28 = 15 +;; @0025 v29 = iconst.i64 -16 +;; @0025 v31 = band v30, v29 ; v29 = -16 +;; @0025 v33 = iadd v32, v31 +;; @0025 v34 = uextend.i64 v26 +;; @0025 v35 = icmp ule v33, v34 +;; @0025 brif v35, block2, block3 ;; ;; block2: -;; v134 = iconst.i32 15 -;; v135 = iadd.i32 v21, v134 ; v134 = 15 -;; v138 = iconst.i32 -16 -;; v139 = band v135, v138 ; v138 = -16 -;; v141 = iadd.i32 v24, v139 -;; @0025 store notrap aligned region0 v141, v23 -;; v155 = iconst.i32 -1476395002 -;; v156 = load.i64 notrap aligned readonly can_move v0+8 -;; v157 = load.i64 notrap aligned readonly can_move v156+32 -;; @0025 v48 = iadd v157, v31 -;; @0025 store notrap aligned v155, v48 ; v155 = -1476395002 -;; v158 = load.i64 notrap aligned readonly can_move v0+40 -;; v159 = load.i32 notrap aligned readonly can_move v158 -;; @0025 store notrap aligned v159, v48+4 -;; v160 = band.i64 v29, v28 ; v28 = -16 -;; @0025 istore32 notrap aligned v160, v48+8 -;; @0025 jump block4(v24, v48) +;; v136 = iconst.i32 15 +;; v137 = iadd.i32 v22, v136 ; v136 = 15 +;; v140 = iconst.i32 -16 +;; v141 = band v137, v140 ; v140 = -16 +;; v143 = iadd.i32 v25, v141 +;; @0025 store notrap aligned region0 v143, v24 +;; v157 = iconst.i32 -1476395002 +;; v158 = load.i64 notrap aligned readonly can_move v0+8 +;; v159 = load.i64 notrap aligned readonly can_move v158+32 +;; @0025 v49 = iadd v159, v32 +;; @0025 store notrap aligned v157, v49 ; v157 = -1476395002 +;; v160 = load.i64 notrap aligned readonly can_move v0+40 +;; v161 = load.i32 notrap aligned readonly can_move v160 +;; @0025 store notrap aligned v161, v49+4 +;; v162 = band.i64 v30, v29 ; v29 = -16 +;; @0025 istore32 notrap aligned v162, v49+8 +;; @0025 jump block4(v25, v49) ;; ;; block3 cold: -;; @0025 v36 = iconst.i32 -1476395002 -;; @0025 v38 = load.i64 notrap aligned readonly can_move v0+40 -;; @0025 v39 = load.i32 notrap aligned readonly can_move v38 -;; @0025 v40 = iconst.i32 16 -;; @0025 v41 = call fn0(v0, v36, v39, v21, v40) ; v36 = -1476395002, v40 = 16 -;; @0025 v120 = load.i64 notrap aligned readonly can_move v0+8 -;; @0025 v42 = load.i64 notrap aligned readonly can_move v120+32 -;; @0025 v43 = uextend.i64 v41 -;; @0025 v44 = iadd v42, v43 -;; @0025 jump block4(v41, v44) +;; @0025 v37 = iconst.i32 -1476395002 +;; @0025 v39 = load.i64 notrap aligned readonly can_move v0+40 +;; @0025 v40 = load.i32 notrap aligned readonly can_move v39 +;; @0025 v41 = iconst.i32 16 +;; @0025 v42 = call fn0(v0, v37, v40, v22, v41) ; v37 = -1476395002, v41 = 16 +;; @0025 v122 = load.i64 notrap aligned readonly can_move v0+8 +;; @0025 v43 = load.i64 notrap aligned readonly can_move v122+32 +;; @0025 v44 = uextend.i64 v42 +;; @0025 v45 = iadd v43, v44 +;; @0025 jump block4(v42, v45) ;; -;; block4(v53: i32, v54: i64): -;; v119 = stack_addr.i64 ss0 -;; store notrap v53, v119 -;; v118 = iconst.i64 16 -;; @0025 v55 = iadd v54, v118 ; v118 = 16 -;; @0025 store.i32 user2 v3, v55 -;; v99 = load.i32 notrap v119 -;; @0025 trapz v99, user16 -;; v161 = load.i64 notrap aligned readonly can_move v0+8 -;; v162 = load.i64 notrap aligned readonly can_move v161+32 -;; @0025 v57 = uextend.i64 v99 -;; @0025 v59 = iadd v162, v57 -;; @0025 v61 = iadd v59, v118 ; v118 = 16 -;; @0025 v62 = load.i32 user2 readonly v61 -;; @0025 v63 = uextend.i64 v62 -;; @0025 v68 = icmp.i64 ugt v8, v63 -;; @0025 trapnz v68, user17 -;; @0025 v77 = uload32.i64 notrap aligned v0+56 -;; @0025 v82 = icmp.i64 ugt v10, v77 -;; @0025 trapnz v82, heap_oob -;; @0025 v84 = load.i64 notrap aligned v0+48 -;; @0025 v91 = load.i64 notrap aligned v161+40 -;; v109 = iconst.i64 20 -;; @0025 v72 = iadd v59, v109 ; v109 = 20 -;; @0025 v93 = uadd_overflow_trap v72, v8, user2 -;; @0025 v92 = iadd v162, v91 -;; @0025 v94 = icmp ugt v93, v92 -;; @0025 trapnz v94, user2 -;; @0025 v86 = iadd v84, v7 -;; @0025 call fn1(v0, v72, v86, v8), stack_map=[i32 @ ss0+0] -;; v96 = load.i32 notrap v119 +;; block4(v54: i32, v55: i64): +;; v121 = stack_addr.i64 ss0 +;; store notrap v54, v121 +;; v120 = iconst.i64 16 +;; @0025 v56 = iadd v55, v120 ; v120 = 16 +;; @0025 store.i32 user2 v3, v56 +;; v101 = load.i32 notrap v121 +;; @0025 trapz v101, user16 +;; v163 = load.i64 notrap aligned readonly can_move v0+8 +;; v164 = load.i64 notrap aligned readonly can_move v163+32 +;; @0025 v58 = uextend.i64 v101 +;; @0025 v60 = iadd v164, v58 +;; @0025 v62 = iadd v60, v120 ; v120 = 16 +;; @0025 v63 = load.i32 user2 readonly v62 +;; @0025 v64 = uextend.i64 v63 +;; @0025 v69 = icmp.i64 ugt v9, v64 +;; @0025 trapnz v69, user17 +;; @0025 v78 = load.i32 notrap aligned v0+56 +;; @0025 v79 = uextend.i64 v78 +;; @0025 v84 = icmp.i64 ugt v11, v79 +;; @0025 trapnz v84, heap_oob +;; @0025 v86 = load.i64 notrap aligned v0+48 +;; @0025 v93 = load.i64 notrap aligned v163+40 +;; v111 = iconst.i64 20 +;; @0025 v73 = iadd v60, v111 ; v111 = 20 +;; @0025 v95 = uadd_overflow_trap v73, v9, user2 +;; @0025 v94 = iadd v164, v93 +;; @0025 v96 = icmp ugt v95, v94 +;; @0025 trapnz v96, user2 +;; @0025 v88 = iadd v86, v8 +;; @0025 call fn1(v0, v73, v88, v9), stack_map=[i32 @ ss0+0] +;; v98 = load.i32 notrap v121 ;; @0029 jump block1 ;; ;; block1: -;; @0029 return v96 +;; @0029 return v98 ;; } diff --git a/tests/disas/gc/array-new-default-anyref.wat b/tests/disas/gc/array-new-default-anyref.wat index cccc027faadf..c80b3f392509 100644 --- a/tests/disas/gc/array-new-default-anyref.wat +++ b/tests/disas/gc/array-new-default-anyref.wat @@ -10,3 +10,113 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 2 "vmctx" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 +;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 +;; gv6 = load.i64 notrap aligned gv4+40 +;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail +;; fn0 = colocated u805306368:24 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @001f v5 = uextend.i64 v2 +;; v98 = iconst.i64 2 +;; v99 = ishl v5, v98 ; v98 = 2 +;; v96 = iconst.i64 32 +;; @001f v7 = ushr v99, v96 ; v96 = 32 +;; @001f trapnz v7, user18 +;; @001f v4 = iconst.i32 20 +;; v105 = iconst.i32 2 +;; v106 = ishl v2, v105 ; v105 = 2 +;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20 +;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 +;; @001f v12 = load.i32 notrap aligned v11 +;; @001f v13 = load.i32 notrap aligned v11+4 +;; @001f v19 = uextend.i64 v12 +;; @001f v14 = uextend.i64 v9 +;; @001f v15 = iconst.i64 15 +;; @001f v17 = iadd v14, v15 ; v15 = 15 +;; @001f v16 = iconst.i64 -16 +;; @001f v18 = band v17, v16 ; v16 = -16 +;; @001f v20 = iadd v19, v18 +;; @001f v21 = uextend.i64 v13 +;; @001f v22 = icmp ule v20, v21 +;; @001f brif v22, block2, block3 +;; +;; block2: +;; v114 = iconst.i32 15 +;; v115 = iadd.i32 v9, v114 ; v114 = 15 +;; v118 = iconst.i32 -16 +;; v119 = band v115, v118 ; v118 = -16 +;; v121 = iadd.i32 v12, v119 +;; @001f store notrap aligned region0 v121, v11 +;; v137 = iconst.i32 -1476394994 +;; v138 = load.i64 notrap aligned readonly can_move v0+8 +;; v139 = load.i64 notrap aligned readonly can_move v138+32 +;; @001f v36 = iadd v139, v19 +;; @001f store notrap aligned v137, v36 ; v137 = -1476394994 +;; v140 = load.i64 notrap aligned readonly can_move v0+40 +;; v141 = load.i32 notrap aligned readonly can_move v140 +;; @001f store notrap aligned v141, v36+4 +;; v142 = band.i64 v17, v16 ; v16 = -16 +;; @001f istore32 notrap aligned v142, v36+8 +;; @001f jump block4(v12, v36) +;; +;; block3 cold: +;; @001f v24 = iconst.i32 -1476394994 +;; @001f v26 = load.i64 notrap aligned readonly can_move v0+40 +;; @001f v27 = load.i32 notrap aligned readonly can_move v26 +;; @001f v28 = iconst.i32 16 +;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 +;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8 +;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32 +;; @001f v31 = uextend.i64 v29 +;; @001f v32 = iadd v30, v31 +;; @001f jump block4(v29, v32) +;; +;; block4(v41: i32, v42: i64): +;; v91 = iconst.i64 16 +;; @001f v43 = iadd v42, v91 ; v91 = 16 +;; @001f store.i32 user2 v2, v43 +;; @001f trapz v41, user16 +;; v143 = load.i64 notrap aligned readonly can_move v0+8 +;; v144 = load.i64 notrap aligned readonly can_move v143+32 +;; @001f v46 = uextend.i64 v41 +;; @001f v48 = iadd v144, v46 +;; @001f v50 = iadd v48, v91 ; v91 = 16 +;; @001f v51 = load.i32 user2 readonly v50 +;; @001f v52 = uextend.i64 v51 +;; @001f v57 = icmp.i64 ugt v5, v52 +;; @001f trapnz v57, user17 +;; @001f v68 = load.i64 notrap aligned v143+40 +;; v85 = iconst.i64 20 +;; @001f v61 = iadd v48, v85 ; v85 = 20 +;; @001f v70 = uadd_overflow_trap v61, v99, user2 +;; @001f v69 = iadd v144, v68 +;; @001f v71 = icmp ugt v70, v69 +;; @001f trapnz v71, user2 +;; v123 = iconst.i64 0 +;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0 +;; @001f v44 = iconst.i32 0 +;; v97 = iconst.i64 4 +;; @001f v72 = iadd v61, v99 +;; @001f brif v73, block6, block5(v61) +;; +;; block5(v74: i64): +;; v145 = iconst.i32 0 +;; @001f store user2 little v145, v74 ; v145 = 0 +;; v146 = iconst.i64 4 +;; v147 = iadd v74, v146 ; v146 = 4 +;; @001f v76 = icmp eq v147, v72 +;; @001f brif v76, block6, block5(v147) +;; +;; block6: +;; @0022 jump block1(v41) +;; +;; block1(v3: i32): +;; @0022 return v3 +;; } diff --git a/tests/disas/gc/array-new-default-exnref.wat b/tests/disas/gc/array-new-default-exnref.wat index ee921a2c8709..10e6a6d2a3bd 100644 --- a/tests/disas/gc/array-new-default-exnref.wat +++ b/tests/disas/gc/array-new-default-exnref.wat @@ -10,3 +10,113 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 2 "vmctx" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 +;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 +;; gv6 = load.i64 notrap aligned gv4+40 +;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail +;; fn0 = colocated u805306368:24 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @001f v5 = uextend.i64 v2 +;; v98 = iconst.i64 2 +;; v99 = ishl v5, v98 ; v98 = 2 +;; v96 = iconst.i64 32 +;; @001f v7 = ushr v99, v96 ; v96 = 32 +;; @001f trapnz v7, user18 +;; @001f v4 = iconst.i32 20 +;; v105 = iconst.i32 2 +;; v106 = ishl v2, v105 ; v105 = 2 +;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20 +;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 +;; @001f v12 = load.i32 notrap aligned v11 +;; @001f v13 = load.i32 notrap aligned v11+4 +;; @001f v19 = uextend.i64 v12 +;; @001f v14 = uextend.i64 v9 +;; @001f v15 = iconst.i64 15 +;; @001f v17 = iadd v14, v15 ; v15 = 15 +;; @001f v16 = iconst.i64 -16 +;; @001f v18 = band v17, v16 ; v16 = -16 +;; @001f v20 = iadd v19, v18 +;; @001f v21 = uextend.i64 v13 +;; @001f v22 = icmp ule v20, v21 +;; @001f brif v22, block2, block3 +;; +;; block2: +;; v114 = iconst.i32 15 +;; v115 = iadd.i32 v9, v114 ; v114 = 15 +;; v118 = iconst.i32 -16 +;; v119 = band v115, v118 ; v118 = -16 +;; v121 = iadd.i32 v12, v119 +;; @001f store notrap aligned region0 v121, v11 +;; v137 = iconst.i32 -1476394994 +;; v138 = load.i64 notrap aligned readonly can_move v0+8 +;; v139 = load.i64 notrap aligned readonly can_move v138+32 +;; @001f v36 = iadd v139, v19 +;; @001f store notrap aligned v137, v36 ; v137 = -1476394994 +;; v140 = load.i64 notrap aligned readonly can_move v0+40 +;; v141 = load.i32 notrap aligned readonly can_move v140 +;; @001f store notrap aligned v141, v36+4 +;; v142 = band.i64 v17, v16 ; v16 = -16 +;; @001f istore32 notrap aligned v142, v36+8 +;; @001f jump block4(v12, v36) +;; +;; block3 cold: +;; @001f v24 = iconst.i32 -1476394994 +;; @001f v26 = load.i64 notrap aligned readonly can_move v0+40 +;; @001f v27 = load.i32 notrap aligned readonly can_move v26 +;; @001f v28 = iconst.i32 16 +;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 +;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8 +;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32 +;; @001f v31 = uextend.i64 v29 +;; @001f v32 = iadd v30, v31 +;; @001f jump block4(v29, v32) +;; +;; block4(v41: i32, v42: i64): +;; v91 = iconst.i64 16 +;; @001f v43 = iadd v42, v91 ; v91 = 16 +;; @001f store.i32 user2 v2, v43 +;; @001f trapz v41, user16 +;; v143 = load.i64 notrap aligned readonly can_move v0+8 +;; v144 = load.i64 notrap aligned readonly can_move v143+32 +;; @001f v46 = uextend.i64 v41 +;; @001f v48 = iadd v144, v46 +;; @001f v50 = iadd v48, v91 ; v91 = 16 +;; @001f v51 = load.i32 user2 readonly v50 +;; @001f v52 = uextend.i64 v51 +;; @001f v57 = icmp.i64 ugt v5, v52 +;; @001f trapnz v57, user17 +;; @001f v68 = load.i64 notrap aligned v143+40 +;; v85 = iconst.i64 20 +;; @001f v61 = iadd v48, v85 ; v85 = 20 +;; @001f v70 = uadd_overflow_trap v61, v99, user2 +;; @001f v69 = iadd v144, v68 +;; @001f v71 = icmp ugt v70, v69 +;; @001f trapnz v71, user2 +;; v123 = iconst.i64 0 +;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0 +;; @001f v44 = iconst.i32 0 +;; v97 = iconst.i64 4 +;; @001f v72 = iadd v61, v99 +;; @001f brif v73, block6, block5(v61) +;; +;; block5(v74: i64): +;; v145 = iconst.i32 0 +;; @001f store user2 little v145, v74 ; v145 = 0 +;; v146 = iconst.i64 4 +;; v147 = iadd v74, v146 ; v146 = 4 +;; @001f v76 = icmp eq v147, v72 +;; @001f brif v76, block6, block5(v147) +;; +;; block6: +;; @0022 jump block1(v41) +;; +;; block1(v3: i32): +;; @0022 return v3 +;; } diff --git a/tests/disas/gc/array-new-default-externref.wat b/tests/disas/gc/array-new-default-externref.wat index 4bad3865c1f5..bb5b2a4e564d 100644 --- a/tests/disas/gc/array-new-default-externref.wat +++ b/tests/disas/gc/array-new-default-externref.wat @@ -10,3 +10,113 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 2 "vmctx" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 +;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 +;; gv6 = load.i64 notrap aligned gv4+40 +;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail +;; fn0 = colocated u805306368:24 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @001f v5 = uextend.i64 v2 +;; v98 = iconst.i64 2 +;; v99 = ishl v5, v98 ; v98 = 2 +;; v96 = iconst.i64 32 +;; @001f v7 = ushr v99, v96 ; v96 = 32 +;; @001f trapnz v7, user18 +;; @001f v4 = iconst.i32 20 +;; v105 = iconst.i32 2 +;; v106 = ishl v2, v105 ; v105 = 2 +;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20 +;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 +;; @001f v12 = load.i32 notrap aligned v11 +;; @001f v13 = load.i32 notrap aligned v11+4 +;; @001f v19 = uextend.i64 v12 +;; @001f v14 = uextend.i64 v9 +;; @001f v15 = iconst.i64 15 +;; @001f v17 = iadd v14, v15 ; v15 = 15 +;; @001f v16 = iconst.i64 -16 +;; @001f v18 = band v17, v16 ; v16 = -16 +;; @001f v20 = iadd v19, v18 +;; @001f v21 = uextend.i64 v13 +;; @001f v22 = icmp ule v20, v21 +;; @001f brif v22, block2, block3 +;; +;; block2: +;; v114 = iconst.i32 15 +;; v115 = iadd.i32 v9, v114 ; v114 = 15 +;; v118 = iconst.i32 -16 +;; v119 = band v115, v118 ; v118 = -16 +;; v121 = iadd.i32 v12, v119 +;; @001f store notrap aligned region0 v121, v11 +;; v137 = iconst.i32 -1476394994 +;; v138 = load.i64 notrap aligned readonly can_move v0+8 +;; v139 = load.i64 notrap aligned readonly can_move v138+32 +;; @001f v36 = iadd v139, v19 +;; @001f store notrap aligned v137, v36 ; v137 = -1476394994 +;; v140 = load.i64 notrap aligned readonly can_move v0+40 +;; v141 = load.i32 notrap aligned readonly can_move v140 +;; @001f store notrap aligned v141, v36+4 +;; v142 = band.i64 v17, v16 ; v16 = -16 +;; @001f istore32 notrap aligned v142, v36+8 +;; @001f jump block4(v12, v36) +;; +;; block3 cold: +;; @001f v24 = iconst.i32 -1476394994 +;; @001f v26 = load.i64 notrap aligned readonly can_move v0+40 +;; @001f v27 = load.i32 notrap aligned readonly can_move v26 +;; @001f v28 = iconst.i32 16 +;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 +;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8 +;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32 +;; @001f v31 = uextend.i64 v29 +;; @001f v32 = iadd v30, v31 +;; @001f jump block4(v29, v32) +;; +;; block4(v41: i32, v42: i64): +;; v91 = iconst.i64 16 +;; @001f v43 = iadd v42, v91 ; v91 = 16 +;; @001f store.i32 user2 v2, v43 +;; @001f trapz v41, user16 +;; v143 = load.i64 notrap aligned readonly can_move v0+8 +;; v144 = load.i64 notrap aligned readonly can_move v143+32 +;; @001f v46 = uextend.i64 v41 +;; @001f v48 = iadd v144, v46 +;; @001f v50 = iadd v48, v91 ; v91 = 16 +;; @001f v51 = load.i32 user2 readonly v50 +;; @001f v52 = uextend.i64 v51 +;; @001f v57 = icmp.i64 ugt v5, v52 +;; @001f trapnz v57, user17 +;; @001f v68 = load.i64 notrap aligned v143+40 +;; v85 = iconst.i64 20 +;; @001f v61 = iadd v48, v85 ; v85 = 20 +;; @001f v70 = uadd_overflow_trap v61, v99, user2 +;; @001f v69 = iadd v144, v68 +;; @001f v71 = icmp ugt v70, v69 +;; @001f trapnz v71, user2 +;; v123 = iconst.i64 0 +;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0 +;; @001f v44 = iconst.i32 0 +;; v97 = iconst.i64 4 +;; @001f v72 = iadd v61, v99 +;; @001f brif v73, block6, block5(v61) +;; +;; block5(v74: i64): +;; v145 = iconst.i32 0 +;; @001f store user2 little v145, v74 ; v145 = 0 +;; v146 = iconst.i64 4 +;; v147 = iadd v74, v146 ; v146 = 4 +;; @001f v76 = icmp eq v147, v72 +;; @001f brif v76, block6, block5(v147) +;; +;; block6: +;; @0022 jump block1(v41) +;; +;; block1(v3: i32): +;; @0022 return v3 +;; } diff --git a/tests/disas/gc/array-new-default-funcref.wat b/tests/disas/gc/array-new-default-funcref.wat index bb794de1c79b..45901103e3ca 100644 --- a/tests/disas/gc/array-new-default-funcref.wat +++ b/tests/disas/gc/array-new-default-funcref.wat @@ -41,3 +41,90 @@ ;; @001f v13 = load.i32 notrap aligned v11+4 ;; @001f v19 = uextend.i64 v12 ;; @001f v14 = uextend.i64 v9 +;; @001f v15 = iconst.i64 15 +;; @001f v17 = iadd v14, v15 ; v15 = 15 +;; @001f v16 = iconst.i64 -16 +;; @001f v18 = band v17, v16 ; v16 = -16 +;; @001f v20 = iadd v19, v18 +;; @001f v21 = uextend.i64 v13 +;; @001f v22 = icmp ule v20, v21 +;; @001f brif v22, block2, block3 +;; +;; block2: +;; v126 = iconst.i32 15 +;; v127 = iadd.i32 v9, v126 ; v126 = 15 +;; v130 = iconst.i32 -16 +;; v131 = band v127, v130 ; v130 = -16 +;; v133 = iadd.i32 v12, v131 +;; @001f store notrap aligned region0 v133, v11 +;; v148 = iconst.i32 -1476395002 +;; v149 = load.i64 notrap aligned readonly can_move v0+8 +;; v150 = load.i64 notrap aligned readonly can_move v149+32 +;; @001f v36 = iadd v150, v19 +;; @001f store notrap aligned v148, v36 ; v148 = -1476395002 +;; v151 = load.i64 notrap aligned readonly can_move v0+40 +;; v152 = load.i32 notrap aligned readonly can_move v151 +;; @001f store notrap aligned v152, v36+4 +;; v153 = band.i64 v17, v16 ; v16 = -16 +;; @001f istore32 notrap aligned v153, v36+8 +;; @001f jump block4(v12, v36) +;; +;; block3 cold: +;; @001f v24 = iconst.i32 -1476395002 +;; @001f v26 = load.i64 notrap aligned readonly can_move v0+40 +;; @001f v27 = load.i32 notrap aligned readonly can_move v26 +;; @001f v28 = iconst.i32 16 +;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476395002, v28 = 16 +;; @001f v104 = load.i64 notrap aligned readonly can_move v0+8 +;; @001f v30 = load.i64 notrap aligned readonly can_move v104+32 +;; @001f v31 = uextend.i64 v29 +;; @001f v32 = iadd v30, v31 +;; @001f jump block4(v29, v32) +;; +;; block4(v41: i32, v42: i64): +;; v103 = stack_addr.i64 ss0 +;; store notrap v41, v103 +;; v102 = iconst.i64 16 +;; @001f v43 = iadd v42, v102 ; v102 = 16 +;; @001f store.i32 user2 v2, v43 +;; v83 = load.i32 notrap v103 +;; @001f trapz v83, user16 +;; v154 = load.i64 notrap aligned readonly can_move v0+8 +;; v155 = load.i64 notrap aligned readonly can_move v154+32 +;; @001f v46 = uextend.i64 v83 +;; @001f v48 = iadd v155, v46 +;; @001f v50 = iadd v48, v102 ; v102 = 16 +;; @001f v51 = load.i32 user2 readonly v50 +;; @001f v52 = uextend.i64 v51 +;; @001f v57 = icmp.i64 ugt v5, v52 +;; @001f trapnz v57, user17 +;; @001f v68 = load.i64 notrap aligned v154+40 +;; v93 = iconst.i64 20 +;; @001f v61 = iadd v48, v93 ; v93 = 20 +;; @001f v70 = uadd_overflow_trap v61, v111, user2 +;; @001f v69 = iadd v155, v68 +;; @001f v71 = icmp ugt v70, v69 +;; @001f trapnz v71, user2 +;; @001f v44 = iconst.i64 0 +;; @001f v73 = icmp.i64 eq v5, v44 ; v44 = 0 +;; v109 = iconst.i64 4 +;; @001f v72 = iadd v61, v111 +;; @001f brif v73, block6, block5(v61) +;; +;; block5(v74: i64): +;; v156 = iconst.i64 0 +;; @001f v76 = call fn1(v0, v156), stack_map=[i32 @ ss0+0] ; v156 = 0 +;; @001f v77 = ireduce.i32 v76 +;; @001f store user2 little v77, v74 +;; v157 = iconst.i64 4 +;; v158 = iadd v74, v157 ; v157 = 4 +;; @001f v79 = icmp eq v158, v72 +;; @001f brif v79, block6, block5(v158) +;; +;; block6: +;; v80 = load.i32 notrap v103 +;; @0022 jump block1 +;; +;; block1: +;; @0022 return v80 +;; } diff --git a/tests/disas/passive-data.wat b/tests/disas/passive-data.wat index 0fbe3b45fcbe..0d8605e9c419 100644 --- a/tests/disas/passive-data.wat +++ b/tests/disas/passive-data.wat @@ -28,29 +28,30 @@ ;; @003d v6 = load.i64 notrap aligned v0+64 ;; @003d v7 = uextend.i64 v2 ;; @003d v8 = uextend.i64 v4 -;; v32 = iconst.i64 1 -;; @003d v9 = imul v8, v32 ; v32 = 1 +;; v33 = iconst.i64 1 +;; @003d v9 = imul v8, v33 ; v33 = 1 ;; @003d v10 = iadd v7, v9 ;; @003d v11 = icmp ugt v10, v6 ;; @003d trapnz v11, heap_oob ;; @003d v12 = load.i64 notrap aligned readonly can_move v0+56 ;; @003d v13 = uextend.i64 v2 -;; v30 = iconst.i64 1 -;; @003d v14 = imul v13, v30 ; v30 = 1 +;; v31 = iconst.i64 1 +;; @003d v14 = imul v13, v31 ; v31 = 1 ;; @003d v15 = iadd v12, v14 -;; @003d v17 = uload32 notrap aligned v0+152 -;; @003d v18 = uextend.i64 v3 -;; @003d v19 = uextend.i64 v4 -;; v29 = iconst.i64 1 -;; @003d v20 = imul v19, v29 ; v29 = 1 -;; @003d v21 = iadd v18, v20 -;; @003d v22 = icmp ugt v21, v17 -;; @003d trapnz v22, heap_oob -;; @003d v24 = load.i64 notrap aligned v0+144 -;; @003d v25 = uextend.i64 v3 -;; @003d v26 = iadd v24, v25 -;; @003d v27 = uextend.i64 v4 -;; @003d call fn0(v0, v15, v26, v27) +;; @003d v17 = load.i32 notrap aligned v0+152 +;; @003d v18 = uextend.i64 v17 +;; @003d v19 = uextend.i64 v3 +;; @003d v20 = uextend.i64 v4 +;; v30 = iconst.i64 1 +;; @003d v21 = imul v20, v30 ; v30 = 1 +;; @003d v22 = iadd v19, v21 +;; @003d v23 = icmp ugt v22, v18 +;; @003d trapnz v23, heap_oob +;; @003d v25 = load.i64 notrap aligned v0+144 +;; @003d v26 = uextend.i64 v3 +;; @003d v27 = iadd v25, v26 +;; @003d v28 = uextend.i64 v4 +;; @003d call fn0(v0, v15, v27, v28) ;; @0041 jump block1 ;; ;; block1: diff --git a/tests/disas/startup-data-active.wat b/tests/disas/startup-data-active.wat new file mode 100644 index 000000000000..95e2e5b187b0 --- /dev/null +++ b/tests/disas/startup-data-active.wat @@ -0,0 +1,62 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' +;;! flags = '-Wgc -Wfunction-references' + +(module + (memory 1) + + (data (i32.const 1) "hi") +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned gv0+64 +;; gv2 = load.i64 notrap aligned readonly can_move gv0+56 +;; sig0 = (i64 vmctx, i64, i64, i64) tail +;; fn0 = colocated u805306368:1 sig0 +;; +;; block0(v0: i64, v1: i64): +;; v3 = load.i64 notrap aligned v0+112 +;; v4 = iconst.i64 0 +;; v5 = icmp eq v3, v4 ; v4 = 0 +;; brif v5, block2, block1 +;; +;; block1: +;; v8 = load.i32 notrap aligned v0+120 +;; v11 = load.i64 notrap aligned v0+64 +;; v13 = uextend.i64 v8 +;; v16 = icmp ugt v13, v11 +;; trapnz v16, heap_oob +;; v17 = load.i64 notrap aligned readonly can_move v0+56 +;; call fn0(v0, v17, v3, v13) +;; jump block2 +;; +;; block2: +;; return +;; } diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat new file mode 100644 index 000000000000..10e6af2848d3 --- /dev/null +++ b/tests/disas/startup-elem-active.wat @@ -0,0 +1,77 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' +;;! flags = '-Wgc -Wfunction-references' + +(module + (table 10 anyref) + + (elem (i32.const 1) (ref i31) + (item (ref.i31 (i32.const 10))) + (item (ref.i31 (i32.const 11))) + (item (ref.i31 (i32.const 12))) + ) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; region0 = 1 "table" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned gv0+48 +;; gv2 = load.i64 notrap aligned gv0+56 +;; +;; block0(v0: i64, v1: i64): +;; v4 = load.i64 notrap aligned v0+56 +;; v5 = ireduce.i32 v4 +;; v6 = uextend.i64 v5 +;; v86 = iconst.i64 4 +;; v92 = icmp ult v6, v86 ; v86 = 4 +;; trapnz v92, user6 +;; v12 = load.i64 notrap aligned v0+48 +;; v103 = iconst.i32 21 +;; v2 = iconst.i32 1 +;; v114 = icmp ule v5, v2 ; v2 = 1 +;; v79 = iconst.i64 0 +;; v15 = iadd v12, v86 ; v86 = 4 +;; v28 = select_spectre_guard v114, v79, v15 ; v79 = 0 +;; store user6 aligned region0 v103, v28 ; v103 = 21 +;; v117 = iconst.i32 23 +;; v123 = iconst.i32 2 +;; v129 = icmp ule v5, v123 ; v123 = 2 +;; v131 = iconst.i64 8 +;; v39 = iadd v12, v131 ; v131 = 8 +;; v41 = select_spectre_guard v129, v79, v39 ; v79 = 0 +;; store user6 aligned region0 v117, v41 ; v117 = 23 +;; v133 = iconst.i32 25 +;; v3 = iconst.i32 3 +;; v144 = icmp ule v5, v3 ; v3 = 3 +;; v146 = iconst.i64 12 +;; v52 = iadd v12, v146 ; v146 = 12 +;; v54 = select_spectre_guard v144, v79, v52 ; v79 = 0 +;; store user6 aligned region0 v133, v54 ; v133 = 25 +;; return +;; } diff --git a/tests/disas/startup-global.wat b/tests/disas/startup-global.wat new file mode 100644 index 000000000000..49da0905dfc8 --- /dev/null +++ b/tests/disas/startup-global.wat @@ -0,0 +1,42 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' + +(module + (global i64 i64.const 0 i64.const 0 i64.add) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; region0 = 1 "table" +;; gv0 = vmctx +;; +;; block0(v0: i64, v1: i64): +;; v2 = iconst.i64 0 +;; store notrap aligned region0 v2, v0+48 ; v2 = 0 +;; return +;; } diff --git a/tests/disas/startup-passive-segment.wat b/tests/disas/startup-passive-segment.wat new file mode 100644 index 000000000000..eaec68a40327 --- /dev/null +++ b/tests/disas/startup-passive-segment.wat @@ -0,0 +1,50 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' +;;! flags = '-Wgc' + +(module + (elem (ref i31) (item (ref.i31 (i32.const 0))) (item (ref.i31 (i32.const 1)))) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; gv0 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; fn0 = colocated u805306368:4 sig0 +;; +;; block0(v0: i64, v1: i64): +;; v3 = iconst.i32 0 +;; v4 = call fn0(v0, v3) ; v3 = 0 +;; v18 = iconst.i32 1 +;; store user2 little v18, v4 ; v18 = 1 +;; v25 = iconst.i32 3 +;; v13 = iconst.i64 16 +;; v12 = iadd v4, v13 ; v13 = 16 +;; store user2 little v25, v12 ; v25 = 3 +;; return +;; } diff --git a/tests/disas/startup-start.wat b/tests/disas/startup-start.wat new file mode 100644 index 000000000000..aac06f24462c --- /dev/null +++ b/tests/disas/startup-start.wat @@ -0,0 +1,42 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' + +(module + (func $f) + (start $f) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u0:0 sig0 +;; +;; block0(v0: i64, v1: i64): +;; call fn0(v0, v0) +;; return +;; } diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat new file mode 100644 index 000000000000..657a42ecbb54 --- /dev/null +++ b/tests/disas/startup-table-initial-value.wat @@ -0,0 +1,65 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' +;;! flags = '-Wgc -Wfunction-references' + +(module + (table 10 (ref i31) (ref.i31 (i32.const 0))) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned gv0+48 +;; gv2 = load.i64 notrap aligned gv0+56 +;; +;; block0(v0: i64, v1: i64): +;; v7 = load.i64 notrap aligned v0+56 +;; v8 = ireduce.i32 v7 +;; v9 = uextend.i64 v8 +;; v41 = iconst.i64 10 +;; v53 = icmp ult v9, v41 ; v41 = 10 +;; trapnz v53, user6 +;; v15 = load.i64 notrap aligned v0+48 +;; v34 = iconst.i32 1 +;; v83 = iconst.i64 36 +;; v85 = iadd v15, v83 ; v83 = 36 +;; v29 = iconst.i64 4 +;; jump block1(v15) +;; +;; block1(v23: i64): +;; v88 = iconst.i32 1 +;; store notrap aligned v88, v23 ; v88 = 1 +;; v89 = iadd.i64 v15, v83 ; v83 = 36 +;; v90 = icmp eq v23, v89 +;; v91 = iconst.i64 4 +;; v92 = iadd v23, v91 ; v91 = 4 +;; brif v90, block2, block1(v92) +;; +;; block2: +;; return +;; } diff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.wat index ae2e8eaf4363..e9ccc89ad7c3 100644 --- a/tests/disas/winch/x64/table/init_copy_drop.wat +++ b/tests/disas/winch/x64/table/init_copy_drop.wat @@ -303,7 +303,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -388,7 +388,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -473,7 +473,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -558,7 +558,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -643,7 +643,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -754,7 +754,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 0xc(%rsp), %edx -;; callq 0x114d +;; callq 0x1178 ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index be5eaefc49e5..ffa64484d01c 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -1722,7 +1722,7 @@ where self.emit_bounds_check_and_compute_addr(&dst_heap, dst_raw_addr, dst.reg, len.reg)?; self.context.free_reg(dst); - let passive_data_index = match self.env.translation.passive_data_map[segment] { + let runtime_data_index = match self.env.translation.runtime_data_map[segment] { Some(i) => i, // Active data segments always have length zero, so this is only @@ -1747,7 +1747,7 @@ where let data_segment_length_offset = self .env .vmoffsets - .vmctx_passive_data_length(passive_data_index); + .vmctx_runtime_data_length(runtime_data_index); let tmp1 = self.context.any_gpr(self.masm)?; let tmp2 = self.context.any_gpr(self.masm)?; self.masm.load( @@ -1774,7 +1774,7 @@ where let data_segment_base_offset = self .env .vmoffsets - .vmctx_passive_data_base(passive_data_index); + .vmctx_runtime_data_base(runtime_data_index); self.masm.load( self.masm.address_at_vmctx(data_segment_base_offset)?, writable!(tmp1), @@ -1804,7 +1804,7 @@ where } pub fn emit_data_drop(&mut self, data_index: DataIndex) -> Result<()> { - let passive_data_index = match self.env.translation.passive_data_map[data_index] { + let runtime_data_index = match self.env.translation.runtime_data_map[data_index] { Some(idx) => idx, // Active data segments do nothing when dropped, so this is a noop. None => return Ok(()), @@ -1812,7 +1812,7 @@ where let data_segment_offset = self .env .vmoffsets - .vmctx_passive_data_length(passive_data_index); + .vmctx_runtime_data_length(runtime_data_index); let len_addr = self.masm.address_at_vmctx(data_segment_offset)?; self.masm.store(RegImm::i32(0), len_addr, OperandSize::S32) } From 1f3f64a6984d67838da9da8317f6c60c0ac8ca95 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 27 May 2026 18:21:18 -0700 Subject: [PATCH 4/4] Review comments --- crates/cranelift/src/compiler.rs | 4 ++-- crates/cranelift/src/func_environ.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index b3fd3941185e..5bc500550328 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -648,7 +648,7 @@ impl wasmtime_environ::Compiler for Compiler { }; let ty = types[ty].unwrap_func(); match abi { - // This is a think array-to-wasm shim around the actual + // This is a thin array-to-wasm shim around the actual // implementation. Abi::Array => self.array_to_wasm_trampoline( key, @@ -658,7 +658,7 @@ impl wasmtime_environ::Compiler for Compiler { self.isa.pointer_bytes().vmctx_store_context().into(), wasmtime_environ::VMCONTEXT_MAGIC, ), - // Delegate to a helper t o finish compiling this. + // Delegate to a helper to finish compiling this. Abi::Wasm => self.compile_module_startup(translation, types, key, ty), Abi::Patchable => unreachable!(), } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 7f711037cac5..c6f97a67a545 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1483,13 +1483,13 @@ impl FuncEnvironment<'_> { if !self.module.globals[index].mutability { if let Some(index) = self.module.defined_global_index(index) { - if let Some((_, value)) = self + if let Ok(i) = self .module .global_initializers - .iter() - .find(|(def_index, _)| *def_index == index) + .binary_search_by_key(&index, |(def_index, _)| *def_index) { - return GlobalVariable::Constant { value: *value }; + let (_, value) = self.module.global_initializers[i]; + return GlobalVariable::Constant { value }; } } }