diff --git a/crates/core/src/slab.rs b/crates/core/src/slab.rs index 38a483a839f9..030d6294a2ea 100644 --- a/crates/core/src/slab.rs +++ b/crates/core/src/slab.rs @@ -390,15 +390,11 @@ impl Slab { /// /// Returns `None` if the value has since been deallocated. /// - /// If `id` comes from a different `Slab` instance, this method may panic, - /// return `None`, or return an arbitrary value. + /// If `id` comes from a different `Slab` instance, this method may return + /// `None`, or return an arbitrary value. #[inline] pub fn get(&self, id: Id) -> Option<&T> { - match self - .entries - .get(id.0.index()) - .expect("id from different slab") - { + match self.entries.get(id.0.index())? { Entry::Occupied(x) => Some(x), Entry::Free { .. } => None, } @@ -408,15 +404,11 @@ impl Slab { /// /// Returns `None` if the value has since been deallocated. /// - /// If `id` comes from a different `Slab` instance, this method may panic, - /// return `None`, or return an arbitrary value. + /// If `id` comes from a different `Slab` instance, this method may return + /// `None`, or return an arbitrary value. #[inline] pub fn get_mut(&mut self, id: Id) -> Option<&mut T> { - match self - .entries - .get_mut(id.0.index()) - .expect("id from different slab") - { + match self.entries.get_mut(id.0.index())? { Entry::Occupied(x) => Some(x), Entry::Free { .. } => None, } diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index a2c76c9f8a89..5bc500550328 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -39,9 +39,9 @@ use wasmtime_environ::{ Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody, DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder, FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, HostCall, Inlining, - InliningCompiler, ModulePC, ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapSection, - StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, WasmFuncType, - WasmValType, prelude::*, + InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize, + StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, + WasmFuncType, WasmValType, prelude::*, }; use wasmtime_unwinder::ExceptionTableBuilder; @@ -199,6 +199,230 @@ impl Compiler { builder.ins().call_indirect(sig, addr, args) } + + fn compile_wasm_to_array_trampoline( + &self, + wasm_func_ty: &WasmFuncType, + key: FuncKey, + symbol: &str, + ) -> Result { + log::trace!("compiling wasm-to-array trampoline: {key:?} = {symbol:?}"); + + let isa = &*self.isa; + let pointer_type = isa.pointer_type(); + let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables); + let array_call_sig = array_call_signature(isa); + + let mut compiler = self.function_compiler(); + let func = ir::Function::with_name_signature(key_to_name(key), wasm_call_sig); + let (mut builder, block0) = compiler.builder(func); + + let args = builder.func.dfg.block_params(block0).to_vec(); + let callee_vmctx = args[0]; + let caller_vmctx = args[1]; + + // We are exiting Wasm, so save our PC and FP. + // + // Assert that the caller vmctx really is a core Wasm vmctx, since + // that's what we are assuming with our offsets below. + self.debug_assert_vmctx_kind( + &mut builder, + caller_vmctx, + wasmtime_environ::VMCONTEXT_MAGIC, + ); + let ptr = isa.pointer_bytes(); + let vm_store_context = builder.ins().load( + pointer_type, + MemFlagsData::trusted(), + caller_vmctx, + i32::from(ptr.vmcontext_store_context()), + ); + save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr, vm_store_context); + + // Spill all wasm arguments to the stack in `ValRaw` slots. + let (args_base, args_len) = + self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]); + let args_len = builder.ins().iconst(pointer_type, i64::from(args_len)); + + // Load the actual callee out of the + // `VMArrayCallHostFuncContext::host_func`. + let ptr_size = isa.pointer_bytes(); + let callee = builder.ins().load( + pointer_type, + MemFlagsData::trusted(), + callee_vmctx, + ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(), + ); + + // Do an indirect call to the callee. + let callee_signature = builder.func.import_signature(array_call_sig); + let call = self.call_indirect_host( + &mut builder, + HostCall::ArrayCall, + callee_signature, + callee, + &[callee_vmctx, caller_vmctx, args_base, args_len], + ); + + // Increment the "execution version" on the VMStoreContext if + // guest debugging is enabled. + if self.tunables.debug_guest { + let vmstore_ctx_ptr = builder.ins().load( + pointer_type, + MemFlagsData::trusted().with_readonly(), + caller_vmctx, + i32::from(ptr_size.vmctx_store_context()), + ); + let old_version = builder.ins().load( + ir::types::I64, + MemFlagsData::trusted(), + vmstore_ctx_ptr, + i32::from(ptr_size.vmstore_context_execution_version()), + ); + let new_version = builder.ins().iadd_imm(old_version, 1); + builder.ins().store( + MemFlagsData::trusted(), + new_version, + vmstore_ctx_ptr, + i32::from(ptr_size.vmstore_context_execution_version()), + ); + } + + // Invoke `raise` if the callee (host) returned an error. + let succeeded = builder.func.dfg.inst_results(call)[0]; + self.raise_if_host_trapped(&mut builder, caller_vmctx, succeeded); + + // Return results from the array as native return values. + let results = + self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len); + builder.ins().return_(&results); + builder.finalize(); + + Ok(CompiledFunctionBody { + code: box_dyn_any_compiler_context(Some(compiler.cx)), + needs_gc_heap: false, + }) + } + + fn compile_wasm_to_builtin( + &self, + key: FuncKey, + symbol: &str, + ) -> Result { + log::trace!("compiling wasm-to-builtin trampoline: {key:?} = {symbol:?}"); + + let isa = &*self.isa; + let ptr_size = isa.pointer_bytes(); + let pointer_type = isa.pointer_type(); + let sigs = BuiltinFunctionSignatures::new(self); + + let (builtin_func_index, wasm_sig) = match key { + FuncKey::WasmToBuiltinTrampoline(builtin) => (builtin, sigs.wasm_signature(builtin)), + FuncKey::PatchableToBuiltinTrampoline(builtin) => { + let mut sig = sigs.wasm_signature(builtin); + // Patchable functions cannot return anything. We + // raise any errors that occur below so this is fine. + sig.returns.clear(); + sig.call_conv = CallConv::PreserveAll; + (builtin, sig) + } + _ => unreachable!(), + }; + let host_sig = sigs.host_signature(builtin_func_index); + + let mut compiler = self.function_compiler(); + let func = ir::Function::with_name_signature(key_to_name(key), wasm_sig.clone()); + let (mut builder, block0) = compiler.builder(func); + let vmctx = builder.block_params(block0)[0]; + + // Debug-assert that this is the right kind of vmctx, and then + // additionally perform the "routine of the exit trampoline" of saving + // fp/pc/etc. + self.debug_assert_vmctx_kind(&mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC); + let vm_store_context = builder.ins().load( + pointer_type, + MemFlagsData::trusted(), + vmctx, + ptr_size.vmcontext_store_context(), + ); + save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, vm_store_context); + + // Now it's time to delegate to the actual builtin. Forward all our own + // arguments to the libcall itself. + let args = builder.block_params(block0).to_vec(); + let call = self.call_builtin(&mut builder, vmctx, &args, builtin_func_index, host_sig); + let results = builder.func.dfg.inst_results(call).to_vec(); + + // Libcalls do not explicitly jump/raise on traps but instead return a + // code indicating whether they trapped or not. This means that it's the + // responsibility of the trampoline to check for an trapping return + // value and raise a trap as appropriate. With the `results` above check + // what `index` is and for each libcall that has a trapping return value + // process it here. + match builtin_func_index.trap_sentinel() { + Some(TrapSentinel::Falsy) => { + self.raise_if_host_trapped(&mut builder, vmctx, results[0]); + } + Some(TrapSentinel::NegativeTwo) => { + let ty = builder.func.dfg.value_type(results[0]); + let trapped = builder.ins().iconst(ty, -2); + let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped); + self.raise_if_host_trapped(&mut builder, vmctx, succeeded); + } + Some(TrapSentinel::Negative) => { + let ty = builder.func.dfg.value_type(results[0]); + let zero = builder.ins().iconst(ty, 0); + let succeeded = + builder + .ins() + .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero); + self.raise_if_host_trapped(&mut builder, vmctx, succeeded); + } + Some(TrapSentinel::NegativeOne) => { + let ty = builder.func.dfg.value_type(results[0]); + let minus_one = builder.ins().iconst(ty, -1); + let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one); + self.raise_if_host_trapped(&mut builder, vmctx, succeeded); + } + None => {} + } + + // And finally, return all the results of this libcall. + if !wasm_sig.returns.is_empty() { + builder.ins().return_(&results); + } else { + builder.ins().return_(&[]); + } + builder.finalize(); + + Ok(CompiledFunctionBody { + code: box_dyn_any_compiler_context(Some(compiler.cx)), + needs_gc_heap: false, + }) + } + + fn compile_module_startup( + &self, + translation: &ModuleTranslation<'_>, + types: &ModuleTypesBuilder, + key: FuncKey, + ty: &WasmFuncType, + ) -> Result { + let mut compiler = self.function_compiler(); + let context = &mut compiler.cx.codegen_context; + context.func.signature = wasm_call_signature(&*self.isa, ty, &self.tunables); + let (namespace, index) = key.into_raw_parts(); + context.func.name = UserFuncName::User(UserExternalName { namespace, index }); + let mut func_env = FuncEnvironment::new(self, translation, types, ty, key); + compiler + .cx + .func_translator + .translate_module_startup(&mut context.func, &mut func_env)?; + Ok(CompiledFunctionBody { + code: box_dyn_any_compiler_context(Some(compiler.cx)), + needs_gc_heap: func_env.needs_gc_heap(), + }) + } } fn box_dyn_any_compiled_function(f: CompiledFunction) -> Box { @@ -364,130 +588,82 @@ impl wasmtime_environ::Compiler for Compiler { }) } - fn compile_array_to_wasm_trampoline( + fn compile_trampoline( &self, - translation: &ModuleTranslation<'_>, - types: &ModuleTypesBuilder, - key: FuncKey, - symbol: &str, - ) -> Result { - let (module_index, def_func_index) = key.unwrap_array_to_wasm_trampoline(); - let func_index = translation.module.func_index(def_func_index); - let sig = translation.module.functions[func_index] - .signature - .unwrap_module_type_index(); - self.array_to_wasm_trampoline( - key, - FuncKey::DefinedWasmFunction(module_index, def_func_index), - types[sig].unwrap_func(), - symbol, - self.isa.pointer_bytes().vmctx_store_context().into(), - wasmtime_environ::VMCONTEXT_MAGIC, - ) - } - - fn compile_wasm_to_array_trampoline( - &self, - wasm_func_ty: &WasmFuncType, + translation: Option<&ModuleTranslation<'_>>, key: FuncKey, + types: &ModuleTypesBuilder, symbol: &str, ) -> Result { - log::trace!("compiling wasm-to-array trampoline: {key:?} = {symbol:?}"); - - let isa = &*self.isa; - let pointer_type = isa.pointer_type(); - let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables); - let array_call_sig = array_call_signature(isa); - - let mut compiler = self.function_compiler(); - let func = ir::Function::with_name_signature(key_to_name(key), wasm_call_sig); - let (mut builder, block0) = compiler.builder(func); + match key { + FuncKey::ArrayToWasmTrampoline(module_index, def_func_index) => { + let translation = translation.unwrap(); + let func_index = translation.module.func_index(def_func_index); + let sig = translation.module.functions[func_index] + .signature + .unwrap_module_type_index(); + self.array_to_wasm_trampoline( + key, + FuncKey::DefinedWasmFunction(module_index, def_func_index), + types[sig].unwrap_func(), + symbol, + self.isa.pointer_bytes().vmctx_store_context().into(), + wasmtime_environ::VMCONTEXT_MAGIC, + ) + } - let args = builder.func.dfg.block_params(block0).to_vec(); - let callee_vmctx = args[0]; - let caller_vmctx = args[1]; + FuncKey::WasmToArrayTrampoline(ty) => { + let ty = types[ty].unwrap_func(); + self.compile_wasm_to_array_trampoline(ty, key, symbol) + } + #[cfg(feature = "component-model")] + FuncKey::ResourceDropTrampoline => { + let ty = types.find_resource_drop_signature().unwrap(); + let ty = types[ty].unwrap_func(); + self.compile_wasm_to_array_trampoline(ty, key, symbol) + } - // We are exiting Wasm, so save our PC and FP. - // - // Assert that the caller vmctx really is a core Wasm vmctx, since - // that's what we are assuming with our offsets below. - self.debug_assert_vmctx_kind( - &mut builder, - caller_vmctx, - wasmtime_environ::VMCONTEXT_MAGIC, - ); - let ptr = isa.pointer_bytes(); - let vm_store_context = builder.ins().load( - pointer_type, - MemFlagsData::trusted(), - caller_vmctx, - i32::from(ptr.vmcontext_store_context()), - ); - save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr, vm_store_context); + FuncKey::DefinedWasmFunction(..) => { + panic!("use `compile_function` instead") + } - // Spill all wasm arguments to the stack in `ValRaw` slots. - let (args_base, args_len) = - self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]); - let args_len = builder.ins().iconst(pointer_type, i64::from(args_len)); + FuncKey::WasmToBuiltinTrampoline(..) | FuncKey::PatchableToBuiltinTrampoline(_) => { + self.compile_wasm_to_builtin(key, symbol) + } - // Load the actual callee out of the - // `VMArrayCallHostFuncContext::host_func`. - let ptr_size = isa.pointer_bytes(); - let callee = builder.ins().load( - pointer_type, - MemFlagsData::trusted(), - callee_vmctx, - ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(), - ); + FuncKey::PulleyHostCall(_) => unreachable!(), - // Do an indirect call to the callee. - let callee_signature = builder.func.import_signature(array_call_sig); - let call = self.call_indirect_host( - &mut builder, - HostCall::ArrayCall, - callee_signature, - callee, - &[callee_vmctx, caller_vmctx, args_base, args_len], - ); + #[cfg(feature = "component-model")] + FuncKey::ComponentTrampoline(..) | FuncKey::UnsafeIntrinsic(..) => { + unreachable!() + } - // Increment the "execution version" on the VMStoreContext if - // guest debugging is enabled. - if self.tunables.debug_guest { - let vmstore_ctx_ptr = builder.ins().load( - pointer_type, - MemFlagsData::trusted().with_readonly(), - caller_vmctx, - i32::from(ptr_size.vmctx_store_context()), - ); - let old_version = builder.ins().load( - ir::types::I64, - MemFlagsData::trusted(), - vmstore_ctx_ptr, - i32::from(ptr_size.vmstore_context_execution_version()), - ); - let new_version = builder.ins().iadd_imm(old_version, 1); - builder.ins().store( - MemFlagsData::trusted(), - new_version, - vmstore_ctx_ptr, - i32::from(ptr_size.vmstore_context_execution_version()), - ); + FuncKey::ModuleStartup(abi, module) => { + let translation = translation.unwrap(); + let ty = match translation.module.startup { + ModuleStartup::None => unreachable!(), + ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => { + t.unwrap_module_type_index() + } + }; + let ty = types[ty].unwrap_func(); + match abi { + // This is a thin array-to-wasm shim around the actual + // implementation. + Abi::Array => self.array_to_wasm_trampoline( + key, + FuncKey::ModuleStartup(Abi::Wasm, module), + ty, + symbol, + self.isa.pointer_bytes().vmctx_store_context().into(), + wasmtime_environ::VMCONTEXT_MAGIC, + ), + // Delegate to a helper to finish compiling this. + Abi::Wasm => self.compile_module_startup(translation, types, key, ty), + Abi::Patchable => unreachable!(), + } + } } - - // Invoke `raise` if the callee (host) returned an error. - let succeeded = builder.func.dfg.inst_results(call)[0]; - self.raise_if_host_trapped(&mut builder, caller_vmctx, succeeded); - - // Return results from the array as native return values. - let results = - self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len); - builder.ins().return_(&results); - builder.finalize(); - - Ok(CompiledFunctionBody { - code: box_dyn_any_compiler_context(Some(compiler.cx)), - needs_gc_heap: false, - }) } fn append_code( @@ -752,103 +928,6 @@ impl wasmtime_environ::Compiler for Compiler { self.isa.create_systemv_cie() } - fn compile_wasm_to_builtin( - &self, - key: FuncKey, - symbol: &str, - ) -> Result { - log::trace!("compiling wasm-to-builtin trampoline: {key:?} = {symbol:?}"); - - let isa = &*self.isa; - let ptr_size = isa.pointer_bytes(); - let pointer_type = isa.pointer_type(); - let sigs = BuiltinFunctionSignatures::new(self); - - let (builtin_func_index, wasm_sig) = match key { - FuncKey::WasmToBuiltinTrampoline(builtin) => (builtin, sigs.wasm_signature(builtin)), - FuncKey::PatchableToBuiltinTrampoline(builtin) => { - let mut sig = sigs.wasm_signature(builtin); - // Patchable functions cannot return anything. We - // raise any errors that occur below so this is fine. - sig.returns.clear(); - sig.call_conv = CallConv::PreserveAll; - (builtin, sig) - } - _ => unreachable!(), - }; - let host_sig = sigs.host_signature(builtin_func_index); - - let mut compiler = self.function_compiler(); - let func = ir::Function::with_name_signature(key_to_name(key), wasm_sig.clone()); - let (mut builder, block0) = compiler.builder(func); - let vmctx = builder.block_params(block0)[0]; - - // Debug-assert that this is the right kind of vmctx, and then - // additionally perform the "routine of the exit trampoline" of saving - // fp/pc/etc. - self.debug_assert_vmctx_kind(&mut builder, vmctx, wasmtime_environ::VMCONTEXT_MAGIC); - let vm_store_context = builder.ins().load( - pointer_type, - MemFlagsData::trusted(), - vmctx, - ptr_size.vmcontext_store_context(), - ); - save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, vm_store_context); - - // Now it's time to delegate to the actual builtin. Forward all our own - // arguments to the libcall itself. - let args = builder.block_params(block0).to_vec(); - let call = self.call_builtin(&mut builder, vmctx, &args, builtin_func_index, host_sig); - let results = builder.func.dfg.inst_results(call).to_vec(); - - // Libcalls do not explicitly jump/raise on traps but instead return a - // code indicating whether they trapped or not. This means that it's the - // responsibility of the trampoline to check for an trapping return - // value and raise a trap as appropriate. With the `results` above check - // what `index` is and for each libcall that has a trapping return value - // process it here. - match builtin_func_index.trap_sentinel() { - Some(TrapSentinel::Falsy) => { - self.raise_if_host_trapped(&mut builder, vmctx, results[0]); - } - Some(TrapSentinel::NegativeTwo) => { - let ty = builder.func.dfg.value_type(results[0]); - let trapped = builder.ins().iconst(ty, -2); - let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped); - self.raise_if_host_trapped(&mut builder, vmctx, succeeded); - } - Some(TrapSentinel::Negative) => { - let ty = builder.func.dfg.value_type(results[0]); - let zero = builder.ins().iconst(ty, 0); - let succeeded = - builder - .ins() - .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero); - self.raise_if_host_trapped(&mut builder, vmctx, succeeded); - } - Some(TrapSentinel::NegativeOne) => { - let ty = builder.func.dfg.value_type(results[0]); - let minus_one = builder.ins().iconst(ty, -1); - let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one); - self.raise_if_host_trapped(&mut builder, vmctx, succeeded); - } - None => {} - } - - // And finally, return all the results of this libcall. - if !wasm_sig.returns.is_empty() { - builder.ins().return_(&results); - } else { - builder.ins().return_(&[]); - } - builder.finalize(); - - Ok(CompiledFunctionBody { - code: box_dyn_any_compiler_context(Some(compiler.cx)), - needs_gc_heap: false, - }) - } - fn compiled_function_relocation_targets<'a>( &'a self, func: &'a dyn Any, diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index ec972bb7b340..d10c1370d396 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -1588,7 +1588,7 @@ impl TranslateTrap for TrapTranslator<'_> { } impl ComponentCompiler for Compiler { - fn compile_trampoline( + fn compile_component_trampoline( &self, component: &ComponentTranslation, types: &ComponentTypesBuilder, diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index b3568e4070a4..c6f97a67a545 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -13,7 +13,7 @@ use crate::{ }; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; -use cranelift_codegen::ir::immediates::{Imm64, Offset32, V128Imm}; +use cranelift_codegen::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, V128Imm}; use cranelift_codegen::ir::{ self, BlockArg, Endianness, ExceptionTableData, ExceptionTableItem, types, }; @@ -29,10 +29,12 @@ use std::mem; use wasmparser::{FuncValidator, Operator, WasmFeatures, WasmModuleResources}; use wasmtime_core::math::f64_cvt_to_int_bounds; use wasmtime_environ::{ - BuiltinFunctionIndex, ComponentPC, DataIndex, DefinedFuncIndex, ElemIndex, - EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, - GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module, - ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex, + BuiltinFunctionIndex, ComponentPC, ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, + DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex, + FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex, + IndexType, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables, Module, + ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PassiveElemIndex, PtrSize, + RuntimeDataIndex, Table, TableIndex, TableInitialValue, TableSegment, TableSegmentElements, TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType, }; @@ -1481,8 +1483,12 @@ impl FuncEnvironment<'_> { if !self.module.globals[index].mutability { if let Some(index) = self.module.defined_global_index(index) { - let init = &self.module.global_initializers[index]; - if let Some(value) = init.const_eval() { + if let Ok(i) = self + .module + .global_initializers + .binary_search_by_key(&index, |(def_index, _)| *def_index) + { + let (_, value) = self.module.global_initializers[i]; return GlobalVariable::Constant { value }; } } @@ -2618,7 +2624,7 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let table_data = self.get_or_create_table(builder.func, table_index); let (dst, flags) = table_data.prepare_table_addr(self, builder, index); - self.emit_table_set(builder, table_index, dst, flags, value) + self.emit_table_set(builder, table_index, dst, flags, value, true) } /// Helper to store `value` into the table address at `addr` using `flags`. @@ -2632,19 +2638,32 @@ impl FuncEnvironment<'_> { addr: ir::Value, flags: ir::MemFlagsData, value: ir::Value, + initialized: bool, ) -> WasmResult<()> { let table = self.module.tables[table_index]; match table.ref_type.heap_type.top() { // GC-managed types. WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => { - gc::gc_compiler(self)?.translate_write_gc_reference( - self, - builder, - table.ref_type, - addr, - value, - flags, - ) + let mut gc = gc::gc_compiler(self)?; + if initialized { + gc.translate_write_gc_reference( + self, + builder, + table.ref_type, + addr, + value, + flags, + ) + } else { + gc.translate_init_gc_reference( + self, + builder, + table.ref_type, + addr, + value, + flags, + ) + } } // Function types. @@ -2693,7 +2712,16 @@ impl FuncEnvironment<'_> { val: ir::Value, len: ir::Value, ) -> WasmResult<()> { - self.translate_entity_fill(builder, table_index, dst, val, len) + self.translate_entity_fill( + builder, + CheckedEntity::Table { + table: table_index, + initialized: true, + }, + dst, + val, + len, + ) } pub fn translate_ref_i31( @@ -3164,6 +3192,21 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, global_index: GlobalIndex, val: ir::Value, + ) -> WasmResult<()> { + self.emit_global_set(builder, global_index, val, true) + } + + /// Helper to translate the `global.set` instruction. + /// + /// The main difference with `translate_global_set`, the main entrypoint, is + /// the `initialized` flag to indicate whether the global was previously + /// initialized to a valid value. + fn emit_global_set( + &mut self, + builder: &mut FunctionBuilder<'_>, + global_index: GlobalIndex, + val: ir::Value, + initialized: bool, ) -> WasmResult<()> { match self.get_or_create_global(builder.func, global_index) { GlobalVariable::Constant { .. } => { @@ -3197,14 +3240,26 @@ impl FuncEnvironment<'_> { let gv = builder.ins().global_value(self.pointer_type(), gv); let src = builder.ins().iadd_imm(gv, i64::from(offset)); - gc::gc_compiler(self)?.translate_write_gc_reference( - self, - builder, - ty, - src, - val, - ir::MemFlagsData::trusted(), - )? + let mut gc = gc::gc_compiler(self)?; + if initialized { + gc.translate_write_gc_reference( + self, + builder, + ty, + src, + val, + ir::MemFlagsData::trusted(), + )?; + } else { + gc.translate_init_gc_reference( + self, + builder, + ty, + src, + val, + ir::MemFlagsData::trusted(), + )?; + } } } Ok(()) @@ -3731,7 +3786,7 @@ impl FuncEnvironment<'_> { }, ); } - CheckedEntity::Table(table) => { + CheckedEntity::Table { table, initialized } => { self.emit_raw_array_or_table_fill( builder, entity, @@ -3739,7 +3794,14 @@ impl FuncEnvironment<'_> { val, len_ptr, &|env, builder, addr, value| { - env.emit_table_set(builder, table, addr, ir::MemFlagsData::trusted(), value) + env.emit_table_set( + builder, + table, + addr, + ir::MemFlagsData::trusted(), + value, + initialized, + ) }, )?; } @@ -3761,7 +3823,9 @@ impl FuncEnvironment<'_> { )?; } // Not allowed to be written to in wasm. - CheckedEntity::Data { .. } | CheckedEntity::Elem(_) => unreachable!(), + CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => { + unreachable!() + } } Ok(()) @@ -3993,7 +4057,7 @@ impl FuncEnvironment<'_> { // Lookup the passive data segment corresponding to this data segment. // If this is an active data segment then it already has length 0 so // there's nothing to do. - let passive_index = match self.translation.passive_data_map[seg_index] { + let runtime_index = match self.translation.runtime_data_map[seg_index] { Some(idx) => idx, None => return Ok(()), }; @@ -4006,7 +4070,7 @@ impl FuncEnvironment<'_> { ir::MemFlagsData::trusted(), new_length, vmctx, - i32::try_from(self.offsets.vmctx_passive_data_length(passive_index)).unwrap(), + i32::try_from(self.offsets.vmctx_runtime_data_length(runtime_index)).unwrap(), ); Ok(()) @@ -4080,7 +4144,9 @@ impl FuncEnvironment<'_> { CheckedEntity::Memory(_) => { assert!(matches!( src_entity, - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) )); // A memory's elements are bytes, so the element count is already // the byte length. @@ -4098,7 +4164,7 @@ impl FuncEnvironment<'_> { // Tables/arrays are sometimes a memcpy, sometimes a per-element // loop. Delegate further to figure that out. - CheckedEntity::Table(_) | CheckedEntity::Array { .. } => self + CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => self .emit_raw_array_or_table_copy( builder, dst_entity, @@ -4111,7 +4177,9 @@ impl FuncEnvironment<'_> { ), // Cannot copy into a data or element segment in wasm. - CheckedEntity::Data { .. } | CheckedEntity::Elem(_) => unreachable!(), + CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => { + unreachable!() + } } } @@ -4136,26 +4204,16 @@ impl FuncEnvironment<'_> { // Load the entity size, as `pointer_type`. let entity_size = match entity { CheckedEntity::Memory(i) => self.memory_size_in_bytes(&mut builder.cursor(), i), - CheckedEntity::Table(i) => { - let size = self.translate_table_size(builder.cursor(), i); + CheckedEntity::Table { table, .. } => { + let size = self.translate_table_size(builder.cursor(), table); self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), size, idx_type) } - CheckedEntity::Data { segment, .. } => match self.translation.passive_data_map[segment] + CheckedEntity::Data { segment, .. } => match self.translation.runtime_data_map[segment] { - Some(passive_index) => { - let vmctx = self.vmctx_val(&mut builder.cursor()); - let offset = - i32::try_from(self.offsets.vmctx_passive_data_length(passive_index)) - .unwrap(); - let flags = ir::MemFlagsData::trusted(); - match pointer_type { - I32 => builder.ins().load(I32, flags, vmctx, offset), - I64 => builder.ins().uload32(flags, vmctx, offset), - _ => unreachable!(), - } - } + Some(i) => self.load_runtime_data_length_as_pointer(builder, i), None => builder.ins().iconst(pointer_type, 0), }, + CheckedEntity::RuntimeData(i) => self.load_runtime_data_length_as_pointer(builder, i), CheckedEntity::Elem(i) => match self.translation.passive_elem_map[i] { Some(passive_index) => { let vmctx = self.vmctx_val(&mut builder.cursor()); @@ -4225,30 +4283,20 @@ impl FuncEnvironment<'_> { let heap = &self.heaps()[heap]; builder.ins().global_value(pointer_type, heap.base) } - CheckedEntity::Table(i) => { - let table = self.get_or_create_table(builder.func, i); + CheckedEntity::Table { table, .. } => { + let table = self.get_or_create_table(builder.func, table); builder.ins().global_value(pointer_type, table.base_gv) } - CheckedEntity::Data { segment, .. } => match self.translation.passive_data_map[segment] + CheckedEntity::Data { segment, .. } => match self.translation.runtime_data_map[segment] { - Some(passive_index) => { - let vmctx = self.vmctx_val(&mut builder.cursor()); - let offset = - i32::try_from(self.offsets.vmctx_passive_data_base(passive_index)).unwrap(); - builder.ins().load( - self.pointer_type(), - ir::MemFlagsData::trusted(), - vmctx, - offset, - ) - } - + Some(runtime_index) => self.load_runtime_data_base(builder, runtime_index), // Any address should do for an active data segment, but pick // something non-null for now. Note that the length of an active // data segment is always 0, so we know that the memcpy, if any, // will be 0 elements, so the actual value here doesn't matter. None => builder.ins().iconst(pointer_type, 1), }, + CheckedEntity::RuntimeData(i) => self.load_runtime_data_base(builder, i), // Element segments are quite similar to data segments just above. CheckedEntity::Elem(i) => match self.translation.passive_elem_map[i] { Some(passive_index) => { @@ -4295,6 +4343,41 @@ impl FuncEnvironment<'_> { Ok(builder.ins().iadd(base, byte_offset)) } + fn load_runtime_data_length( + &mut self, + builder: &mut FunctionBuilder<'_>, + runtime_index: RuntimeDataIndex, + ) -> ir::Value { + let vmctx = self.vmctx_val(&mut builder.cursor()); + let offset = i32::try_from(self.offsets.vmctx_runtime_data_length(runtime_index)).unwrap(); + let flags = ir::MemFlagsData::trusted(); + builder.ins().load(I32, flags, vmctx, offset) + } + + fn load_runtime_data_length_as_pointer( + &mut self, + builder: &mut FunctionBuilder<'_>, + runtime_index: RuntimeDataIndex, + ) -> ir::Value { + let length = self.load_runtime_data_length(builder, runtime_index); + self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), length, IndexType::I32) + } + + fn load_runtime_data_base( + &mut self, + builder: &mut FunctionBuilder<'_>, + runtime_index: RuntimeDataIndex, + ) -> ir::Value { + let vmctx = self.vmctx_val(&mut builder.cursor()); + let offset = i32::try_from(self.offsets.vmctx_runtime_data_base(runtime_index)).unwrap(); + builder.ins().load( + self.pointer_type(), + ir::MemFlagsData::trusted(), + vmctx, + offset, + ) + } + pub fn translate_table_copy( &mut self, builder: &mut FunctionBuilder<'_>, @@ -4304,7 +4387,20 @@ impl FuncEnvironment<'_> { src: ir::Value, len: ir::Value, ) -> WasmResult<()> { - self.translate_entity_copy(builder, dst_table_index, src_table_index, dst, src, len) + self.translate_entity_copy( + builder, + CheckedEntity::Table { + table: dst_table_index, + initialized: true, + }, + CheckedEntity::Table { + table: src_table_index, + initialized: true, + }, + dst, + src, + len, + ) } /// Emits a copy between two WebAssembly table or array entities. @@ -4377,7 +4473,7 @@ impl FuncEnvironment<'_> { // would mean that memcpy isn't suitable. If lazy init is // disabled though then funcrefs are just pointers so a // memcpy can be used. - CheckedEntity::Table(_) => self.tunables.table_lazy_init, + CheckedEntity::Table { .. } => self.tunables.table_lazy_init, // The GC heap has integers representing funcrefs, so memcpy // is fine. CheckedEntity::Array { .. } => false, @@ -4385,7 +4481,9 @@ impl FuncEnvironment<'_> { // can't work. CheckedEntity::Elem(_) => true, // Not possible - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => unreachable!(), + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) => unreachable!(), }, }, }; @@ -4439,7 +4537,10 @@ impl FuncEnvironment<'_> { // it's easier right now to share the internals of // `translate_table_get` which are a bit tricky with // funcrefs. - CheckedEntity::Table(i) => this.translate_table_get(builder, i, src_index)?, + CheckedEntity::Table { table, initialized } => { + assert!(initialized); + this.translate_table_get(builder, table, src_index)? + } CheckedEntity::Array { initialized, .. } => { assert!(initialized); let read_ty = src_entity.storage_type(this); @@ -4468,11 +4569,20 @@ impl FuncEnvironment<'_> { } } } - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => unreachable!(), + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) => unreachable!(), }; match dst_entity { - CheckedEntity::Table(i) => { - this.emit_table_set(builder, i, dst, ir::MemFlagsData::trusted(), val)?; + CheckedEntity::Table { table, initialized } => { + this.emit_table_set( + builder, + table, + dst, + ir::MemFlagsData::trusted(), + val, + initialized, + )?; } CheckedEntity::Array { initialized, .. } => { if initialized { @@ -4483,6 +4593,7 @@ impl FuncEnvironment<'_> { } CheckedEntity::Memory(_) | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) | CheckedEntity::Elem(_) => unreachable!(), } Ok(()) @@ -4842,7 +4953,10 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { self.translate_entity_copy( builder, - table_index, + CheckedEntity::Table { + table: table_index, + initialized: true, + }, CheckedEntity::Elem(ElemIndex::from_u32(seg_index)), dst, src, @@ -5645,6 +5759,386 @@ impl FuncEnvironment<'_> { pub fn is_reachable(&self) -> bool { self.stacks.reachable() } + + /// Generates the body of a `FuncKey::ModuleStartup(..)` function. + /// + /// This will perform all initialization of a module before any code in the + /// module itself starts executing. This will, for example, execute all + /// constant expressions for global initializers. + pub fn translate_module_startup(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> { + for (i, expr) in self.translation.global_initializers.iter() { + self.module_initialize_global(builder, *i, expr)?; + } + for (i, exprs) in self.translation.passive_elements.iter() { + self.module_initialize_passive_element(builder, i, exprs)?; + } + for (i, init) in self.translation.table_initialization.initial_values.iter() { + match init { + TableInitialValue::Null => {} + TableInitialValue::Expr(expr) => { + self.module_initialize_table_with_fill(builder, i, expr)?; + } + } + } + for segment in self.translation.table_initialization.segments.iter() { + self.module_initialize_table_with_segment(builder, segment)?; + } + match &self.translation.memory_init { + MemoryInit::Unprocessed(_) => unreachable!(), + MemoryInit::Processed(segments) => { + for (memory, offset, data) in segments.iter() { + self.module_initialize_memory_segment(builder, *memory, offset, *data)?; + } + } + } + if let Some(i) = self.translation.start_func { + self.module_start(builder, i)?; + } + Ok(()) + } + + /// Initializes all "complicated" globals in a module. + /// + /// This will translate the given constant expression and then initialize + /// the global. Note that this translation is done with the global not + /// initialized, meaning that GC barriers are slightly tweaked. + fn module_initialize_global( + &mut self, + builder: &mut FunctionBuilder, + global: DefinedGlobalIndex, + expr: &ConstExpr, + ) -> WasmResult<()> { + let index = self.module.global_index(global); + let val = self.translate_const_expr(builder, expr)?; + self.emit_global_set(builder, index, val, false) + } + + /// Initializes all passive element segments for a module. + /// + /// This will execute all constant expressions for all passive element + /// segments and initialize all `ValRaw` storage on the host. This + /// implements the semantics of how passive element segments are evaluated + /// once in a wasm instance. + fn module_initialize_passive_element( + &mut self, + builder: &mut FunctionBuilder, + elem: PassiveElemIndex, + exprs: &TableSegmentElements, + ) -> WasmResult<()> { + let vmctx = self.vmctx_val(&mut builder.cursor()); + let libcall = self + .builtin_functions + .passive_elem_segment_base(builder.func); + let idx = builder.ins().iconst(I32, i64::from(elem.as_u32())); + let call = builder.ins().call(libcall, &[vmctx, idx]); + let base = builder.func.dfg.first_result(call); + + // Values in `ValRaw` are always stored in little-endian. + let flags = ir::MemFlagsData::trusted().with_endianness(Endianness::Little); + + match exprs { + TableSegmentElements::Functions(indices) => { + for (i, func) in indices.iter().enumerate() { + let func = self.translate_ref_func(builder.cursor(), *func)?; + builder.ins().store( + flags, + func, + base, + i32::try_from(i.checked_mul(16).unwrap()).unwrap(), + ); + } + } + TableSegmentElements::Expressions { exprs, ty } => { + for (i, expr) in exprs.iter().enumerate() { + let val = self.translate_const_expr(builder, expr)?; + + let dst = builder + .ins() + .iadd_imm(base, i64::try_from(i.checked_mul(16).unwrap()).unwrap()); + match ty.heap_type.top() { + WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => { + let ty = WasmStorageType::Val(WasmValType::Ref(*ty)); + gc::init_field_at_addr(self, builder, ty, dst, val)?; + } + WasmHeapTopType::Func | WasmHeapTopType::Cont => { + builder.ins().store( + flags, + val, + base, + i32::try_from(i.checked_mul(16).unwrap()).unwrap(), + ); + } + } + } + } + } + Ok(()) + } + + /// Initializes all tables with non-null initializers. + /// + /// This function will morally execute a `table.fill` for the specified + /// defined table in this module for the result of the constant expression + /// provided. Note that the fill operation is done on an initialized table + /// which slightly alters the barriers emitted for GC types. + fn module_initialize_table_with_fill( + &mut self, + builder: &mut FunctionBuilder, + table: DefinedTableIndex, + init: &ConstExpr, + ) -> WasmResult<()> { + let table = self.module.table_index(table); + let ty = self.module.tables[table]; + let val = self.translate_const_expr(builder, init)?; + let dst = builder.ins().iconst(index_type_to_ir_type(ty.idx_type), 0); + let len = builder.ins().iconst( + index_type_to_ir_type(ty.idx_type), + ty.limits.min.cast_signed(), + ); + self.translate_entity_fill( + builder, + CheckedEntity::Table { + table, + initialized: false, + }, + dst, + val, + len, + ) + } + + /// Executes initialization for an active element segment in a module. + /// + /// Note that this isn't done for "precomputed" tables which won't have + /// initializers present. Additionally note that this doesn't use + /// `table.init` because the element segment won't actually be present + /// anywhere at runtime. + fn module_initialize_table_with_segment( + &mut self, + builder: &mut FunctionBuilder, + segment: &TableSegment, + ) -> WasmResult<()> { + let offset = self.translate_const_expr(builder, &segment.offset)?; + let segment_len = builder.ins().iconst( + index_type_to_ir_type(self.module.tables[segment.table_index].idx_type), + i64::try_from(segment.elements.len()).unwrap(), + ); + + // Check the bounds first before mutating anything. + self.translate_entity_bounds_check( + builder, + CheckedEntity::Table { + table: segment.table_index, + initialized: true, + }, + offset, + segment_len, + )?; + + // Re-use the `table.set` translation for making this a simple function + // to define. That re-executes the bounds check which is a bit + // inefficient, but can be optimized in the future. + match &segment.elements { + TableSegmentElements::Functions(indices) => { + for (i, func) in indices.iter().enumerate() { + let func = self.translate_ref_func(builder.cursor(), *func)?; + let index = builder.ins().iadd_imm(offset, i64::try_from(i).unwrap()); + self.translate_table_set(builder, segment.table_index, func, index)?; + } + } + TableSegmentElements::Expressions { exprs, ty: _ } => { + for (i, expr) in exprs.iter().enumerate() { + let val = self.translate_const_expr(builder, expr)?; + let index = builder.ins().iadd_imm(offset, i64::try_from(i).unwrap()); + self.translate_table_set(builder, segment.table_index, val, index)?; + } + } + } + Ok(()) + } + + /// Peform initialization of an active data segment in a module. + fn module_initialize_memory_segment( + &mut self, + builder: &mut FunctionBuilder, + memory: MemoryIndex, + offset: &MemorySegmentOffset, + data: RuntimeDataIndex, + ) -> WasmResult<()> { + let mut end = None; + let offset = match offset { + // This is a bit subtle, but the goal here is to make a dynamic + // deduction to see if this initialization is actually necessary + // based on state at runtime. With a static initializer the goal is + // to enable CoW initialization at runtime, but not all hosts have + // that configured or supported. If it's supported then we shouldn't + // do anything here, but if it's unsupported then this must actually + // execute initialization. + // + // The host already put the addresses of all data segments into the + // `VMContext` during `VMContext::initialize_vmctx`, and this reads + // out the base pointer. If it's null then the host doesn't actually + // need initialization, so this is skipped, and otherwise it's the + // actual base pointer that's going to be used. + MemorySegmentOffset::Static(n) => { + let current_block = builder.current_block().unwrap(); + let init_block = builder.create_block(); + let end_block = builder.create_block(); + end = Some(end_block); + + builder.ensure_inserted_block(); + builder.insert_block_after(init_block, current_block); + builder.insert_block_after(end_block, init_block); + + let base = self.load_runtime_data_base(builder, data); + let null = builder.ins().iconst(self.pointer_type(), 0); + let is_null = builder.ins().icmp(IntCC::Equal, base, null); + builder.ins().brif(is_null, end_block, &[], init_block, &[]); + builder.switch_to_block(init_block); + builder.seal_block(init_block); + let ty = index_type_to_ir_type(self.module.memories[memory].idx_type); + builder.ins().iconst(ty, n.cast_signed()) + } + MemorySegmentOffset::Expr(expr) => self.translate_const_expr(builder, expr)?, + }; + + // Model the initialization here as a `memory.init`. + let len = self.load_runtime_data_length(builder, data); + let start = builder.ins().iconst(I32, 0); + self.translate_entity_copy(builder, memory, data, offset, start, len)?; + + // Finalize control-flow for the `MemorySegmentOffset::Static` case + // above. + if let Some(end) = end { + builder.ins().jump(end, &[]); + builder.switch_to_block(end); + builder.seal_block(end); + } + Ok(()) + } + + /// Translates the final step of module initialization, executing the + /// `start` function. + fn module_start(&mut self, builder: &mut FunctionBuilder, func: FuncIndex) -> WasmResult<()> { + // Manuall manage fuel around the call as the `Call` opcode does for + // normal wasm to ensure that it's correctly accounted for. + if self.tunables.consume_fuel { + self.fuel_consumed += 1; + self.fuel_increment_var(builder); + self.fuel_save_from_var(builder); + } + let ty = self.module.functions[func] + .signature + .unwrap_module_type_index(); + let sig_ref = self.get_or_create_interned_sig_ref(builder.func, ty); + self.translate_call(builder, Default::default(), func, sig_ref, &[])?; + if self.tunables.consume_fuel { + self.fuel_load_into_var(builder); + } + Ok(()) + } + + /// Helper function to evaluate `expr` into a value. + /// + /// This primarily reuses all the `translate_*` functions elsewhere on this + /// builder toe deduplicate implementation details. + fn translate_const_expr( + &mut self, + builder: &mut FunctionBuilder, + expr: &ConstExpr, + ) -> WasmResult { + let mut stack = Vec::new(); + for op in expr.ops() { + if self.tunables.consume_fuel { + self.fuel_consumed += 1; + } + match op { + ConstOp::I32Const(i) => { + stack.push(builder.ins().iconst(I32, i64::from(*i))); + } + ConstOp::I64Const(i) => { + stack.push(builder.ins().iconst(I64, *i)); + } + ConstOp::F32Const(i) => { + stack.push(builder.ins().f32const(Ieee32::with_bits(*i))); + } + ConstOp::F64Const(i) => { + stack.push(builder.ins().f64const(Ieee64::with_bits(*i))); + } + ConstOp::V128Const(i) => { + let data = i.to_le_bytes().to_vec().into(); + let handle = builder.func.dfg.constants.insert(data); + stack.push(builder.ins().vconst(ir::types::I8X16, handle)); + } + ConstOp::GlobalGet(i) => { + stack.push(self.translate_global_get(builder, *i)?); + } + ConstOp::RefI31 => { + let val = stack.pop().unwrap(); + stack.push(self.translate_ref_i31(builder.cursor(), val)?); + } + ConstOp::RefNull(ty) => { + stack.push(self.translate_ref_null(builder.cursor(), *ty)?); + } + ConstOp::RefFunc(i) => { + stack.push(self.translate_ref_func(builder.cursor(), *i)?); + } + ConstOp::I32Add | ConstOp::I64Add => { + let b = stack.pop().unwrap(); + let a = stack.pop().unwrap(); + stack.push(builder.ins().iadd(a, b)); + } + ConstOp::I32Sub | ConstOp::I64Sub => { + let b = stack.pop().unwrap(); + let a = stack.pop().unwrap(); + stack.push(builder.ins().isub(a, b)); + } + ConstOp::I32Mul | ConstOp::I64Mul => { + let b = stack.pop().unwrap(); + let a = stack.pop().unwrap(); + stack.push(builder.ins().imul(a, b)); + } + ConstOp::StructNew { struct_type_index } => { + let arity = self.struct_fields_len(*struct_type_index)?; + let fields = stack.drain(stack.len() - arity..).collect::<_>(); + stack.push(self.translate_struct_new(builder, *struct_type_index, fields)?); + } + ConstOp::StructNewDefault { struct_type_index } => { + stack.push(self.translate_struct_new_default(builder, *struct_type_index)?); + } + ConstOp::ArrayNew { array_type_index } => { + let len = stack.pop().unwrap(); + let elem = stack.pop().unwrap(); + stack.push(self.translate_array_new(builder, *array_type_index, elem, len)?); + } + ConstOp::ArrayNewDefault { array_type_index } => { + let len = stack.pop().unwrap(); + stack.push(self.translate_array_new_default( + builder, + *array_type_index, + len, + )?); + } + ConstOp::ArrayNewFixed { + array_type_index, + array_size, + } => { + let array_size = usize::try_from(*array_size).unwrap(); + let elems = stack.drain(stack.len() - array_size..).collect::>(); + stack.push(self.translate_array_new_fixed( + builder, + *array_type_index, + &elems, + )?); + } + ConstOp::ExternConvertAny => {} + ConstOp::AnyConvertExtern => {} + } + } + let ret = stack.pop().unwrap(); + assert!(stack.is_empty()); + Ok(ret) + } } // Helper function to convert an `IndexType` to an `ir::Type`. @@ -5698,7 +6192,12 @@ enum CheckedEntity { /// A WebAssembly linear memory. Memory(MemoryIndex), /// A WebAssembly table. - Table(TableIndex), + Table { + /// The index of the table that's being accessed. + table: TableIndex, + /// Whether or not this table has been initialized yet. + initialized: bool, + }, /// A WebAssembly data segment loaded in chunks of `element_size` bytes. /// /// For `memory.init` this will have `element_size = 1`, but for @@ -5711,6 +6210,11 @@ enum CheckedEntity { /// or possibly more for array initialization. element_size: u32, }, + /// A refinement of `Data` above which uses a `RuntimeDataIndex`. + /// + /// This isn't reachable from core wasm but is used during module + /// startup. + RuntimeData(RuntimeDataIndex), /// A WebAssembly passive element segment. Elem(ElemIndex), /// An `arrayref` with the specified type. @@ -5731,9 +6235,9 @@ impl From for CheckedEntity { } } -impl From for CheckedEntity { - fn from(table_index: TableIndex) -> Self { - CheckedEntity::Table(table_index) +impl From for CheckedEntity { + fn from(runtime_data_index: RuntimeDataIndex) -> Self { + CheckedEntity::RuntimeData(runtime_data_index) } } @@ -5742,10 +6246,11 @@ impl CheckedEntity { fn index_type(&self, env: &FuncEnvironment) -> IndexType { match *self { CheckedEntity::Memory(i) => env.memory(i).idx_type, - CheckedEntity::Table(i) => env.table(i).idx_type, - CheckedEntity::Data { .. } | CheckedEntity::Array { .. } | CheckedEntity::Elem(_) => { - IndexType::I32 - } + CheckedEntity::Table { table, .. } => env.table(table).idx_type, + CheckedEntity::Data { .. } + | CheckedEntity::Array { .. } + | CheckedEntity::Elem(_) + | CheckedEntity::RuntimeData(_) => IndexType::I32, } } @@ -5756,9 +6261,9 @@ impl CheckedEntity { func: &mut ir::Function, ) -> WasmResult { Ok(match *self { - CheckedEntity::Memory(_) => 1, + CheckedEntity::Memory(_) | CheckedEntity::RuntimeData(_) => 1, CheckedEntity::Data { element_size, .. } => element_size, - CheckedEntity::Table(table) => env.get_or_create_table(func, table).element_size, + CheckedEntity::Table { table, .. } => env.get_or_create_table(func, table).element_size, CheckedEntity::Array { ty, .. } => env.array_layout(ty)?.elem_size, CheckedEntity::Elem(_) => 16, }) @@ -5770,8 +6275,9 @@ impl CheckedEntity { CheckedEntity::Memory(_) | CheckedEntity::Data { element_size: 1, .. - } => WasmStorageType::I8, - CheckedEntity::Table(table) => { + } + | CheckedEntity::RuntimeData(_) => WasmStorageType::I8, + CheckedEntity::Table { table, .. } => { WasmStorageType::Val(WasmValType::Ref(env.table(table).ref_type)) } CheckedEntity::Array { ty, .. } => { @@ -5786,10 +6292,10 @@ impl CheckedEntity { /// Returns the trap code to use when an index into this entity is out-of-bounds. fn oob_trap_code(&self) -> ir::TrapCode { match self { - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } => { - ir::TrapCode::HEAP_OUT_OF_BOUNDS - } - CheckedEntity::Table(_) | CheckedEntity::Elem(_) => TRAP_TABLE_OUT_OF_BOUNDS, + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) => ir::TrapCode::HEAP_OUT_OF_BOUNDS, + CheckedEntity::Table { .. } | CheckedEntity::Elem(_) => TRAP_TABLE_OUT_OF_BOUNDS, CheckedEntity::Array { .. } => TRAP_ARRAY_OUT_OF_BOUNDS, } } @@ -5798,13 +6304,22 @@ impl CheckedEntity { /// bit patterns. fn allows_memset(&self, env: &FuncEnvironment) -> bool { match self { - CheckedEntity::Memory(_) | CheckedEntity::Data { .. } | CheckedEntity::Array { .. } => { - true + CheckedEntity::Memory(_) + | CheckedEntity::Data { .. } + | CheckedEntity::RuntimeData(_) => true, + CheckedEntity::Array { ty, .. } => { + let array_ty = env.types.unwrap_array(*ty).unwrap(); + // Most GC references need barriers, funcrefs need intern-ing, + // etc. Disallow memset on all reference types. + !matches!( + array_ty.0.element_type, + WasmStorageType::Val(WasmValType::Ref(_)) + ) } CheckedEntity::Elem(_) => false, // Tables that are lazily initialized can't be memset because the // initialized bit needs to be set when storing values. - CheckedEntity::Table(_) => !env.tunables.table_lazy_init, + CheckedEntity::Table { .. } => !env.tunables.table_lazy_init, } } } diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index 9857146f0361..7dd7b5905f42 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -162,6 +162,20 @@ pub trait GcCompiler { flags: ir::MemFlagsData, ) -> WasmResult<()>; + /// Same as [`Self::translate_write_gc_reference`] except that the + /// destination address has not been previously initialized. + /// + /// This will, for example, skip barriers on the destination address. + fn translate_init_gc_reference( + &mut self, + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ty: WasmRefType, + dst: ir::Value, + new_val: ir::Value, + flags: ir::MemFlagsData, + ) -> WasmResult<()>; + /// Stores `val` into `addr` with the `ty` specified. /// /// This initializes a previously-uninitialized field, so barriers on the diff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/enabled/copying.rs index 5c0e89a2445a..79cbe1841e49 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/copying.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/copying.rs @@ -400,6 +400,19 @@ impl GcCompiler for CopyingCompiler { unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) } + fn translate_init_gc_reference( + &mut self, + _func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ty: WasmRefType, + dst: ir::Value, + new_val: ir::Value, + flags: ir::MemFlagsData, + ) -> WasmResult<()> { + // No write barrier needed for the copying collector. + unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) + } + fn init_field( &mut self, func_env: &mut FuncEnvironment<'_>, diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/enabled/drc.rs index 7938257a4b68..d59c3dc6b2dc 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/drc.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/drc.rs @@ -330,109 +330,6 @@ impl DrcCompiler { ); builder.ins().store(GC_MEMFLAGS, new_reserved, ptr, 0); } - - /// Write to an uninitialized GC reference field, initializing it. - /// - /// ```text - /// *dst = new_val - /// ``` - /// - /// Doesn't need to do a full write barrier: we don't have an old reference - /// that is being overwritten and needs its refcount decremented, just a new - /// reference whose count should be incremented. - fn translate_init_gc_reference( - &mut self, - func_env: &mut FuncEnvironment<'_>, - builder: &mut FunctionBuilder, - ty: WasmRefType, - dst: ir::Value, - new_val: ir::Value, - flags: ir::MemFlagsData, - ) -> WasmResult<()> { - let (ref_ty, _) = func_env.reference_type(ty.heap_type); - - // Special case for references to uninhabited bottom types: see - // `translate_write_gc_reference` for details. - if let WasmHeapType::None = ty.heap_type { - if ty.nullable { - let null = builder.ins().iconst(ref_ty, 0); - builder.ins().store(flags, null, dst, 0); - } else { - let zero = builder.ins().iconst(ir::types::I32, 0); - builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT); - } - return Ok(()); - }; - - // Special case for `i31ref`s: no need for any barriers. - if let WasmHeapType::I31 = ty.heap_type { - return unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags); - } - - // Our initialization barrier for GC references being copied out of the - // stack and initializing a table/global/struct field/etc... is roughly - // equivalent to the following pseudo-CLIF: - // - // ``` - // current_block: - // ... - // let new_val_is_null_or_i31 = ... - // brif new_val_is_null_or_i31, continue_block, inc_ref_block - // - // inc_ref_block: - // let ref_count = load new_val.ref_count - // let new_ref_count = iadd_imm ref_count, 1 - // store new_val.ref_count, new_ref_count - // jump check_old_val_block - // - // continue_block: - // store dst, new_val - // ... - // ``` - // - // This write barrier is responsible for ensuring that the new value's - // ref count is incremented now that the table/global/struct/etc... is - // holding onto it. - - let current_block = builder.current_block().unwrap(); - let inc_ref_block = builder.create_block(); - let continue_block = builder.create_block(); - - builder.ensure_inserted_block(); - builder.insert_block_after(inc_ref_block, current_block); - builder.insert_block_after(continue_block, inc_ref_block); - - // Current block: check whether the new value is non-null and - // non-i31. If so, branch to the `inc_ref_block`. - log::trace!("DRC initialization barrier: check if the value is null or i31"); - let new_val_is_null_or_i31 = func_env.gc_ref_is_null_or_i31(builder, ty, new_val); - builder.ins().brif( - new_val_is_null_or_i31, - continue_block, - &[], - inc_ref_block, - &[], - ); - - // Block to increment the ref count of the new value when it is non-null - // and non-i31. - builder.switch_to_block(inc_ref_block); - builder.seal_block(inc_ref_block); - log::trace!("DRC initialization barrier: increment the ref count of the initial value"); - self.mutate_ref_count(func_env, builder, new_val, 1); - builder.ins().jump(continue_block, &[]); - - // Join point after we're done with the GC barrier: do the actual store - // to initialize the field. - builder.switch_to_block(continue_block); - builder.seal_block(continue_block); - log::trace!( - "DRC initialization barrier: finally, store into {dst:?} to initialize the field" - ); - unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)?; - - Ok(()) - } } impl GcCompiler for DrcCompiler { @@ -959,6 +856,109 @@ impl GcCompiler for DrcCompiler { Ok(()) } + /// Write to an uninitialized GC reference field, initializing it. + /// + /// ```text + /// *dst = new_val + /// ``` + /// + /// Doesn't need to do a full write barrier: we don't have an old reference + /// that is being overwritten and needs its refcount decremented, just a new + /// reference whose count should be incremented. + fn translate_init_gc_reference( + &mut self, + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ty: WasmRefType, + dst: ir::Value, + new_val: ir::Value, + flags: ir::MemFlagsData, + ) -> WasmResult<()> { + let (ref_ty, _) = func_env.reference_type(ty.heap_type); + + // Special case for references to uninhabited bottom types: see + // `translate_write_gc_reference` for details. + if let WasmHeapType::None = ty.heap_type { + if ty.nullable { + let null = builder.ins().iconst(ref_ty, 0); + builder.ins().store(flags, null, dst, 0); + } else { + let zero = builder.ins().iconst(ir::types::I32, 0); + builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT); + } + return Ok(()); + }; + + // Special case for `i31ref`s: no need for any barriers. + if let WasmHeapType::I31 = ty.heap_type { + return unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags); + } + + // Our initialization barrier for GC references being copied out of the + // stack and initializing a table/global/struct field/etc... is roughly + // equivalent to the following pseudo-CLIF: + // + // ``` + // current_block: + // ... + // let new_val_is_null_or_i31 = ... + // brif new_val_is_null_or_i31, continue_block, inc_ref_block + // + // inc_ref_block: + // let ref_count = load new_val.ref_count + // let new_ref_count = iadd_imm ref_count, 1 + // store new_val.ref_count, new_ref_count + // jump check_old_val_block + // + // continue_block: + // store dst, new_val + // ... + // ``` + // + // This write barrier is responsible for ensuring that the new value's + // ref count is incremented now that the table/global/struct/etc... is + // holding onto it. + + let current_block = builder.current_block().unwrap(); + let inc_ref_block = builder.create_block(); + let continue_block = builder.create_block(); + + builder.ensure_inserted_block(); + builder.insert_block_after(inc_ref_block, current_block); + builder.insert_block_after(continue_block, inc_ref_block); + + // Current block: check whether the new value is non-null and + // non-i31. If so, branch to the `inc_ref_block`. + log::trace!("DRC initialization barrier: check if the value is null or i31"); + let new_val_is_null_or_i31 = func_env.gc_ref_is_null_or_i31(builder, ty, new_val); + builder.ins().brif( + new_val_is_null_or_i31, + continue_block, + &[], + inc_ref_block, + &[], + ); + + // Block to increment the ref count of the new value when it is non-null + // and non-i31. + builder.switch_to_block(inc_ref_block); + builder.seal_block(inc_ref_block); + log::trace!("DRC initialization barrier: increment the ref count of the initial value"); + self.mutate_ref_count(func_env, builder, new_val, 1); + builder.ins().jump(continue_block, &[]); + + // Join point after we're done with the GC barrier: do the actual store + // to initialize the field. + builder.switch_to_block(continue_block); + builder.seal_block(continue_block); + log::trace!( + "DRC initialization barrier: finally, store into {dst:?} to initialize the field" + ); + unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags)?; + + Ok(()) + } + /// Write to an uninitialized field or element inside a GC object. fn init_field( &mut self, diff --git a/crates/cranelift/src/func_environ/gc/enabled/null.rs b/crates/cranelift/src/func_environ/gc/enabled/null.rs index 70818ab69abe..789bdfb8eac5 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/null.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/null.rs @@ -371,6 +371,18 @@ impl GcCompiler for NullCompiler { unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) } + fn translate_init_gc_reference( + &mut self, + _func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ty: WasmRefType, + dst: ir::Value, + new_val: ir::Value, + flags: ir::MemFlagsData, + ) -> WasmResult<()> { + unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) + } + fn init_field( &mut self, func_env: &mut FuncEnvironment<'_>, diff --git a/crates/cranelift/src/translate/func_translator.rs b/crates/cranelift/src/translate/func_translator.rs index b6875745504f..80323def5c14 100644 --- a/crates/cranelift/src/translate/func_translator.rs +++ b/crates/cranelift/src/translate/func_translator.rs @@ -96,6 +96,26 @@ impl FuncTranslator { log::trace!("translated Wasm to CLIF:\n{}", func.display()); Ok(()) } + + /// Translate the `FuncKey::ModuleStartup` function. + pub fn translate_module_startup( + &mut self, + func: &mut ir::Function, + environ: &mut FuncEnvironment<'_>, + ) -> WasmResult<()> { + let mut builder = FunctionBuilder::new(func, &mut self.func_ctx); + let entry_block = builder.create_block(); + builder.append_block_params_for_function_params(entry_block); + builder.switch_to_block(entry_block); + builder.seal_block(entry_block); + builder.ensure_inserted_block(); + environ.before_translate_function(&mut builder)?; + environ.translate_module_startup(&mut builder)?; + environ.after_translate_function(&mut builder)?; + builder.ins().return_(&[]); + builder.finalize(); + Ok(()) + } } /// Declare local variables for the signature parameters that correspond to WebAssembly locals. diff --git a/crates/environ/src/compile/mod.rs b/crates/environ/src/compile/mod.rs index d3cd69a87234..4dc2fa6d01c3 100644 --- a/crates/environ/src/compile/mod.rs +++ b/crates/environ/src/compile/mod.rs @@ -5,7 +5,7 @@ use crate::error::Result; use crate::prelude::*; use crate::{ DefinedFuncIndex, FlagValue, FuncKey, FunctionLoc, ObjectKind, PrimaryMap, StaticModuleIndex, - TripleExt, Tunables, WasmError, WasmFuncType, obj, + TripleExt, Tunables, WasmError, obj, }; use object::write::{Object, SymbolId}; use object::{Architecture, BinaryFormat, FileFlags}; @@ -261,38 +261,11 @@ pub trait Compiler: Send + Sync { /// /// The trampoline should save the necessary state to record the /// host-to-Wasm transition (e.g. registers used for fast stack walking). - fn compile_array_to_wasm_trampoline( - &self, - translation: &ModuleTranslation<'_>, - types: &ModuleTypesBuilder, - key: FuncKey, - symbol: &str, - ) -> Result; - - /// Compile a trampoline for a Wasm caller calling a array callee with the - /// given signature. - /// - /// The trampoline should save the necessary state to record the - /// Wasm-to-host transition (e.g. registers used for fast stack walking). - fn compile_wasm_to_array_trampoline( - &self, - wasm_func_ty: &WasmFuncType, - key: FuncKey, - symbol: &str, - ) -> Result; - - /// Creates a trampoline that can be used to call Wasmtime's implementation - /// of the builtin function specified by `index`. - /// - /// The trampoline created can technically have any ABI but currently has - /// the native ABI. This will then perform all the necessary duties of an - /// exit trampoline from wasm and then perform the actual dispatch to the - /// builtin function. Builtin functions in Wasmtime are stored in an array - /// in all `VMContext` pointers, so the call to the host is an indirect - /// call. - fn compile_wasm_to_builtin( + fn compile_trampoline( &self, + translation: Option<&ModuleTranslation<'_>>, key: FuncKey, + types: &ModuleTypesBuilder, symbol: &str, ) -> Result; diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rs index ca409dae9791..c9e1f302adfc 100644 --- a/crates/environ/src/compile/module_artifacts.rs +++ b/crates/environ/src/compile/module_artifacts.rs @@ -5,8 +5,7 @@ use crate::WasmChecksum; use crate::error::{Result, bail}; use crate::prelude::*; use crate::{ - CompiledModuleInfo, DebugInfoData, FunctionName, MemoryInitialization, Metadata, - ModuleTranslation, Tunables, obj, + CompiledModuleInfo, DebugInfoData, FunctionName, Metadata, ModuleTranslation, Tunables, obj, }; use object::SectionKind; use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; @@ -118,9 +117,8 @@ impl<'a> ObjectBuilder<'a> { mut module, debuginfo, has_unparsed_debuginfo, - data, data_align, - passive_data, + runtime_data, wasm, .. } = translation; @@ -128,20 +126,15 @@ impl<'a> ObjectBuilder<'a> { // Place all data from the wasm module into a section which will the // source of the data later at runtime. This additionally keeps track of // the offset of - let mut total_data_len = 0; let data_offset = self .obj .append_section_data(self.data, &[], data_align.unwrap_or(1)); - for (i, data) in data.iter().enumerate() { - // The first data segment has its alignment specified as the alignment - // for the entire section, but everything afterwards is adjacent so it - // has alignment of 1. + for (i, (_, data)) in runtime_data.iter().enumerate() { + // The first data segment has its alignment specified as the + // alignment for the entire section, but everything afterwards is + // adjacent so it has alignment of 1. let align = if i == 0 { data_align.unwrap_or(1) } else { 1 }; self.obj.append_section_data(self.data, data, align); - total_data_len += data.len(); - } - for data in passive_data.iter() { - self.obj.append_section_data(self.data, data, 1); } // If any names are present in the module then the `ELF_NAME_DATA` section @@ -172,34 +165,12 @@ impl<'a> ObjectBuilder<'a> { } } - // Data offsets in `MemoryInitialization` are offsets within the - // `translation.data` list concatenated which is now present in the data - // segment that's appended to the object. Increase the offsets by - // `self.data_size` to account for any previously added module. - let data_offset = u32::try_from(data_offset).unwrap(); - match &mut module.memory_initialization { - MemoryInitialization::Segmented(list) => { - for segment in list { - segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); - segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); - } - } - MemoryInitialization::Static { map } => { - for (_, segment) in map { - if let Some(segment) = segment { - segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); - segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); - } - } - } - } - // Data offsets for passive data are relative to the start of - // `translation.passive_data` which was appended to the data segment + // `translation.runtime_data` which was appended to the data segment // of this object, after active data in `translation.data`. Update the // offsets to account prior modules added in addition to active data. - let data_offset = data_offset + u32::try_from(total_data_len).unwrap(); - for (_, range) in module.passive_data.iter_mut() { + let data_offset = u32::try_from(data_offset).unwrap(); + for (_, range) in module.runtime_data.iter_mut() { range.start = range.start.checked_add(data_offset).unwrap(); range.end = range.end.checked_add(data_offset).unwrap(); } diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index be2fc09c0d8b..b5b02b9e6feb 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -1,20 +1,19 @@ use crate::error::{OutOfMemory, Result, bail}; use crate::module::{ - FuncRefIndex, Initializer, MemoryInitialization, MemoryInitializer, Module, TableSegment, - TableSegmentElements, + FuncRefIndex, Initializer, MemoryInitialization, Module, TableSegment, TableSegmentElements, }; use crate::prelude::*; use crate::{ - ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, ElemIndex, EngineOrModuleTypeIndex, - EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType, InitMemory, MemoryIndex, - ModuleInternedTypeIndex, ModuleTypesBuilder, PanicOnOom as _, PassiveDataIndex, - PassiveElemIndex, PrimaryMap, SizeOverflow, StaticMemoryInitializer, StaticModuleIndex, - TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, WasmError, + ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, DefinedGlobalIndex, ElemIndex, + EngineOrModuleTypeIndex, EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType, + MemoryIndex, MemoryInitializer, ModuleInternedTypeIndex, ModuleStartup, ModuleTypesBuilder, + PanicOnOom as _, PassiveElemIndex, PrimaryMap, RuntimeDataIndex, StaticModuleIndex, TableIndex, + TableInitialValue, TableInitialization, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, WasmHeapTopType, WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter, }; +use alloc::borrow::Cow; use cranelift_entity::SecondaryMap; use cranelift_entity::packed_option::ReservedValue; -use std::borrow::Cow; use std::collections::HashMap; use std::mem; use std::path::PathBuf; @@ -84,13 +83,6 @@ pub struct ModuleTranslation<'data> { /// configuration. pub has_unparsed_debuginfo: bool, - /// List of data segments found in this module which should be concatenated - /// together for the final compiled artifact. - /// - /// These data segments, when concatenated, are indexed by the - /// `MemoryInitializer` type. - pub data: Vec>, - /// The desired alignment of `data` in the final data section of the object /// file that we'll emit. /// @@ -100,20 +92,21 @@ pub struct ModuleTranslation<'data> { pub data_align: Option, /// Map from a data segment to whether it's a passive data segment or not. - pub passive_data_map: SecondaryMap>, + pub runtime_data_map: SecondaryMap>, /// Map from an elem segment to whether it's a passive elem segment or not. pub passive_elem_map: SecondaryMap>, - /// Total size of all data pushed onto `data` so far. - total_data: u32, - /// List of passive element segments found in this module which will get /// concatenated for the final artifact. - pub passive_data: Vec<&'data [u8]>, + pub runtime_data: PrimaryMap>, - /// Total size of all passive data pushed into `passive_data` so far. - total_passive_data: u32, + /// Record of all passive data segments that this module contains. + /// + /// These are processed during [`ModuleTranslation::finalize_memory_init`] + /// and eventually moved over into the `runtime_data` list above. Until + /// then, however, their `RuntimeDataIndex` is not yet assigned. + passive_data: Vec<(DataIndex, &'data [u8])>, /// When we're parsing the code section this will be incremented so we know /// which function is currently being defined. @@ -122,6 +115,62 @@ pub struct ModuleTranslation<'data> { /// The type information of the current module made available at the end of the /// validation process. types: Option, + + /// The WebAssembly `start` function, if defined. + pub start_func: Option, + + /// Initializers for `global` values which aren't considered "simple". + /// + /// These initializers are later compiled into a "module startup" function. + pub global_initializers: Vec<(DefinedGlobalIndex, ConstExpr)>, + + /// Definitions of all passive elements found within a module. + /// + /// This maps passive element segments to their definition, either functions + /// or expressions-basd. + pub passive_elements: PrimaryMap, + + /// WebAssembly table initialization data, per table. + /// + /// This keeps track of all per-table initialization (e.g. initial value for + /// non-null tables) as well as active element segments. This is processed + /// and refined by [`ModuleTranslation::finalize_table_init`] after + /// translation. + pub table_initialization: TableInitialization, + + /// WebAssembly memory initialization. + /// + /// This is held here in an `Unprocessed` form during translation, and then + /// this is later finished with [`ModuleTranslation::finalize_memory_init`]. + pub memory_init: MemoryInit<'data>, +} + +/// Different forms of memory initialization that happens for a module. +pub enum MemoryInit<'a> { + /// Raw active data segments that are being applied for an instance. + /// + /// This list contains the raw data which hasn't yet been processed into + /// `RuntimeDataIndex`, for example. This is later processed during + /// [`ModuleTranslation::finalize_memory_init`] to optionally shuffle things + /// around. + Unprocessed(Vec>), + + /// Finalized memory initialization to be executed after + /// [`ModuleTranslation::finalize_memory_init`] has run. This represents + /// active data segments which may have been merged from the `Unprocessed` + /// list above, and may or may not have statically know offsets. + Processed(Vec<(MemoryIndex, MemorySegmentOffset, RuntimeDataIndex)>), +} + +/// Offset within [`MemoryInit::Processed`] which indicates the initial offset +/// a data segment is applied at. +pub enum MemorySegmentOffset { + /// A "complicated" constant expression deferred to get evaluated at runtime + /// with compiled code. + Expr(ConstExpr), + + /// A statically known, in-bounds, constant value. + Static(u64), } impl<'data> ModuleTranslation<'data> { @@ -136,15 +185,18 @@ impl<'data> ModuleTranslation<'data> { exported_signatures: Vec::default(), debuginfo: DebugInfoData::default(), has_unparsed_debuginfo: false, - data: Vec::default(), data_align: None, - total_data: 0, - passive_data: Vec::default(), - total_passive_data: 0, + runtime_data: Default::default(), code_index: 0, types: None, - passive_data_map: Default::default(), + runtime_data_map: Default::default(), passive_elem_map: Default::default(), + start_func: None, + global_initializers: Vec::new(), + passive_elements: Default::default(), + table_initialization: Default::default(), + memory_init: MemoryInit::Unprocessed(Vec::new()), + passive_data: Default::default(), } } @@ -417,9 +469,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.result.module.needs_gc_heap |= table.ref_type.is_vmgcref_type(); self.result.module.tables.push(table)?; let init = match init { - wasmparser::TableInit::RefNull => TableInitialValue::Null { - precomputed: TryVec::new(), - }, + wasmparser::TableInit::RefNull => TableInitialValue::Null, wasmparser::TableInit::Expr(expr) => { let (init, escaped) = ConstExpr::from_wasmparser(self, expr)?; for f in escaped { @@ -428,11 +478,11 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { TableInitialValue::Expr(init) } }; + self.result.table_initialization.initial_values.push(init)?; self.result .module .table_initialization - .initial_values - .push(init)?; + .push(Default::default())?; } } @@ -475,8 +525,24 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.flag_func_escaped(f); } let ty = self.convert_global_type(&ty)?; - self.result.module.globals.push(ty)?; - self.result.module.global_initializers.push(initializer)?; + let index = self.result.module.globals.push(ty)?; + let defined_index = self.result.module.defined_global_index(index).unwrap(); + match initializer.const_eval() { + Some(val) => { + self.result + .module + .global_initializers + .push((defined_index, val))?; + } + None => { + // "Complicated" global initializers are deferred + // to get evaluated in the startup function. + self.require_startup_func(); + self.result + .global_initializers + .push((defined_index, initializer)); + } + } } } @@ -508,9 +574,12 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.validator.start_section(func, &range)?; let func_index = FuncIndex::from_u32(func); - self.flag_func_escaped(func_index); - debug_assert!(self.result.module.start_func.is_none()); - self.result.module.start_func = Some(func_index); + debug_assert!(self.result.start_func.is_none()); + self.result.start_func = Some(func_index); + + // To make startup a bit easier, invoking the `start` function + // is a responsibility deferred to the startup function. + self.require_startup_func(); } Payload::ElementSection(elements) => { @@ -566,19 +635,27 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?; debug_assert!(escaped.is_empty()); - self.result.module.table_initialization.segments.push( - TableSegment { + self.result + .table_initialization + .segments + .push(TableSegment { table_index, offset, elements, - }, - )?; + })?; None } ElementKind::Passive => { - let passive_index = - self.result.module.passive_elements.push(elements)?; + let passive_index = self + .result + .module + .passive_elements + .push((elements.ty(), elements.len()))?; + self.result.passive_elements.push(elements); + // One-time initialization of passive element + // segments is deferred to the startup function. + self.require_startup_func(); Some(passive_index) } @@ -639,14 +716,7 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { Payload::DataSection(data) => { self.validator.data_section(&data)?; - let initializers = match &mut self.result.module.memory_initialization { - MemoryInitialization::Segmented(i) => i, - _ => unreachable!(), - }; - - let cnt = usize::try_from(data.count()).unwrap(); - initializers.reserve_exact(cnt)?; - self.result.data.reserve_exact(cnt); + assert!(self.result.module.memory_initialization.is_segmented()); for (index, entry) in data.into_iter().enumerate() { let wasmparser::Data { @@ -655,53 +725,28 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { range: _, } = entry?; let data_index = DataIndex::from_u32(index.try_into().unwrap()); - let mk_range = |total: &mut u32| -> Result<_, WasmError> { - let range = u32::try_from(data.len()) - .ok() - .and_then(|size| { - let start = *total; - let end = start.checked_add(size)?; - Some(start..end) - }) - .ok_or_else(|| { - WasmError::Unsupported(format!( - "more than 4 gigabytes of data in wasm module", - )) - })?; - *total += range.end - range.start; - Ok(range) - }; - let passive_index = match kind { + match kind { DataKind::Active { memory_index, offset_expr, } => { - let range = mk_range(&mut self.result.total_data)?; let memory_index = MemoryIndex::from_u32(memory_index); let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?; debug_assert!(escaped.is_empty()); - let initializers = match &mut self.result.module.memory_initialization { - MemoryInitialization::Segmented(i) => i, - _ => unreachable!(), + let MemoryInit::Unprocessed(list) = &mut self.result.memory_init else { + panic!("memory initializers should be unprocessed at this point"); }; - initializers.push(MemoryInitializer { + list.push(MemoryInitializer { memory_index, offset, - data: range, - })?; - self.result.data.push(data.into()); - None + data, + }); } DataKind::Passive => { - let range = mk_range(&mut self.result.total_passive_data)?; - self.result.passive_data.push(data); - Some(self.result.module.passive_data.push(range)?) + self.result.passive_data.push((data_index, data)); } - }; - self.result - .passive_data_map - .insert(data_index, passive_index); + } } } @@ -952,6 +997,10 @@ and for re-adding support for interface types you can see this issue: } Ok(()) } + + fn require_startup_func(&mut self) { + self.result.require_startup_func(self.types); + } } impl TypeConvert for ModuleEnvironment<'_, '_> { @@ -971,6 +1020,75 @@ impl TypeConvert for ModuleEnvironment<'_, '_> { } impl ModuleTranslation<'_> { + /// Called after translation is complete this will finalize the memory + /// initialization strategy for this module. + /// + /// This will notably use `Self::try_static_init` to attempt to massage + /// data segments to being CoW-init-friendly. Afterwards the + /// `self.memory_init` field is transitioned from `Unprocessed` to + /// `Processed`. + pub fn finalize_memory_init( + &mut self, + tunables: &Tunables, + page_size: u64, + max_image_size_always_allowed: u64, + types: &mut ModuleTypesBuilder, + ) { + if tunables.memory_init_cow { + self.try_static_init(page_size, max_image_size_always_allowed); + } + + // If any memory is statically initialized, and if that memory has an + // initial data segment, then a startup function is at least + // conditionally needed if the memory needs initialization. Flag as such + // here. + if let MemoryInitialization::Static { map } = &self.module.memory_initialization { + if map.iter().any(|(_, v)| v.is_some()) { + self.require_startup_func_if_memories_need_init(types); + } + } + + // If, after `try_static_init`, initializers are still `Unprocessed` + // then this is the catch-all fallback path for initialization. All + // segments are promoted into `self.runtime_data` and then the + // initialization is rewritten to `Processed`. + if let MemoryInit::Unprocessed(list) = &mut self.memory_init { + let segments = mem::take(list); + let mut new_initializers = Vec::new(); + for segment in segments { + new_initializers.push(( + segment.memory_index, + MemorySegmentOffset::Expr(segment.offset), + self.runtime_data.push(segment.data.into()), + )); + } + if !new_initializers.is_empty() { + self.require_startup_func(types); + } + self.memory_init = MemoryInit::Processed(new_initializers); + } + + // At this point append all passive data to the `runtime_data` list. + // This notably occurs after `try_static_init` above to ensure that the + // page-aligned data for static initialization, if applicable, comes + // first. + for (data_index, segment) in self.passive_data.iter() { + let runtime_index = self.runtime_data.push((*segment).into()); + self.runtime_data_map + .insert(*data_index, Some(runtime_index)); + } + + // And, finally, record all chunks from `self.runtime_data` within + // `self.module.runtime_data` as well. + let mut cur = 0; + for (idx, data) in self.runtime_data.iter() { + let len = u32::try_from(data.len()).unwrap(); + let i = self.module.runtime_data.push(cur..cur + len).panic_on_oom(); + cur += len; + assert_eq!(idx, i); + } + } + /// Attempts to convert segmented memory initialization into static /// initialization for the module that this translation represents. /// @@ -1001,24 +1119,20 @@ impl ModuleTranslation<'_> { /// modules that have some dynamically-placed data segments. But, /// for now, this is sufficient to allow a system that "knows what /// it's doing" to always get static init. - pub fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) { - // This method only attempts to transform a `Segmented` memory init - // into a `Static` one, no other state. - if !self.module.memory_initialization.is_segmented() { - return; - } + fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) { + let segments = match &mut self.memory_init { + MemoryInit::Unprocessed(list) => list, + _ => return, + }; // First a dry run of memory initialization is performed. This // collects information about the extent of memory initialized for each // memory as well as the size of all data segments being copied in. - struct Memory { + struct Memory<'a> { data_size: u64, min_addr: u64, max_addr: u64, - // The `usize` here is a pointer into `self.data` which is the list - // of data segments corresponding to what was found in the original - // wasm module. - segments: Vec<(usize, StaticMemoryInitializer)>, + segments: Vec<(u64, &'a [u8])>, } let mut info = PrimaryMap::with_capacity(self.module.memories.len()); for _ in 0..self.module.memories.len() { @@ -1030,59 +1144,64 @@ impl ModuleTranslation<'_> { }); } - struct InitMemoryAtCompileTime<'a> { - module: &'a Module, - info: &'a mut PrimaryMap, - idx: usize, - } - impl InitMemory for InitMemoryAtCompileTime<'_> { - fn memory_size_in_bytes( - &mut self, - memory_index: MemoryIndex, - ) -> Result { - self.module.memories[memory_index].minimum_byte_size() + for initializer in segments.iter() { + let &MemoryInitializer { + memory_index, + ref offset, + ref data, + } = initializer; + + // Currently `Static` only applies to locally-defined memories, + // so if a data segment references an imported memory then + // transitioning to a `Static` memory initializer is not + // possible. + if self.module.defined_memory_index(memory_index).is_none() { + return; } - fn eval_offset(&mut self, memory_index: MemoryIndex, expr: &ConstExpr) -> Option { - match (expr.ops(), self.module.memories[memory_index].idx_type) { - (&[ConstOp::I32Const(offset)], IndexType::I32) => { - Some(offset.cast_unsigned().into()) + // First up determine the start/end range and verify that they're + // in-bounds for the initial size of the memory at `memory_index`. + // Note that this can bail if we don't have access to globals yet + // (e.g. this is a task happening before instantiation at + // compile-time). + let start = match (offset.ops(), self.module.memories[memory_index].idx_type) { + (&[ConstOp::I32Const(offset)], IndexType::I32) => offset.cast_unsigned().into(), + (&[ConstOp::I64Const(offset)], IndexType::I64) => offset.cast_unsigned(), + _ => return, + }; + let len = u64::try_from(data.len()).unwrap(); + let end = match start.checked_add(len) { + Some(end) => end, + None => return, + }; + + match self.module.memories[memory_index].minimum_byte_size() { + Ok(max) => { + if end > max { + return; } - (&[ConstOp::I64Const(offset)], IndexType::I64) => Some(offset.cast_unsigned()), - _ => None, } + + // Note that computing the minimum can overflow if the page + // size is the default 64KiB and the memory's minimum size in + // pages is `1 << 48`, the maximum number of minimum pages for + // 64-bit memories. We don't return `false` to signal an error + // here and instead defer the error to runtime, when it will be + // impossible to allocate that much memory anyways. + Err(_) => return, } - fn write(&mut self, memory: MemoryIndex, init: &StaticMemoryInitializer) -> bool { - // Currently `Static` only applies to locally-defined memories, - // so if a data segment references an imported memory then - // transitioning to a `Static` memory initializer is not - // possible. - if self.module.defined_memory_index(memory).is_none() { - return false; - }; - let info = &mut self.info[memory]; - let data_len = u64::from(init.data.end - init.data.start); - if data_len > 0 { - info.data_size += data_len; - info.min_addr = info.min_addr.min(init.offset); - info.max_addr = info.max_addr.max(init.offset + data_len); - info.segments.push((self.idx, init.clone())); - } - self.idx += 1; - true + // Skip empty in-bounds data segments. + if data.is_empty() { + continue; } - } - let ok = self - .module - .memory_initialization - .init_memory(&mut InitMemoryAtCompileTime { - idx: 0, - module: &self.module, - info: &mut info, - }); - if !ok { - return; + + let info = &mut info[memory_index]; + let len64 = u64::try_from(data.len()).unwrap(); + info.data_size += len64; + info.min_addr = info.min_addr.min(start); + info.max_addr = info.max_addr.max(start + len64); + info.segments.push((start, data)); } // Validate that the memory information collected is indeed valid for @@ -1127,9 +1246,8 @@ impl ModuleTranslation<'_> { // Here's where we've now committed to changing to static memory. The // memory initialization image is built here from the page data and then // it's converted to a single initializer. - let data = mem::replace(&mut self.data, Vec::new()); let mut map = TryPrimaryMap::with_capacity(info.len()).panic_on_oom(); - let mut module_data_size = 0u32; + let mut new_initializers = Vec::new(); for (memory, info) in info.iter() { // Create the in-memory `image` which is the initialized contents of // this linear memory. @@ -1139,10 +1257,8 @@ impl ModuleTranslation<'_> { 0 }; let mut image = Vec::with_capacity(extent); - for (idx, init) in info.segments.iter() { - let data = &data[*idx]; - assert_eq!(data.len(), init.data.len()); - let offset = usize::try_from(init.offset - info.min_addr).unwrap(); + for (offset, data) in info.segments.iter() { + let offset = usize::try_from(*offset - info.min_addr).unwrap(); if image.len() < offset { image.resize(offset, 0u8); image.extend_from_slice(data); @@ -1181,50 +1297,72 @@ impl ModuleTranslation<'_> { // the front and back with extra zeros as necessary if offset % page_size != 0 { let zero_padding = offset % page_size; - self.data.push(vec![0; zero_padding as usize].into()); + image.splice(0..0, std::iter::repeat(0).take(zero_padding as usize)); offset -= zero_padding; len += zero_padding; } - self.data.push(image.into()); if len % page_size != 0 { let zero_padding = page_size - (len % page_size); - self.data.push(vec![0; zero_padding as usize].into()); + image.extend(std::iter::repeat(0).take(zero_padding as usize)); len += zero_padding; } + let runtime_index = if image.is_empty() { + None + } else { + Some(self.runtime_data.push(image.into())) + }; // Offset/length should now always be page-aligned. assert!(offset % page_size == 0); assert!(len % page_size == 0); - // Create the `StaticMemoryInitializer` which describes this image, + // Record the static memory initializer which describes this image, // only needed if the image is actually present and has a nonzero // length. The `offset` has been calculates above, originally // sourced from `info.min_addr`. The `data` field is the extent // within the final data segment we'll emit to an ELF image, which // is the concatenation of `self.data`, so here it's the size of // the section-so-far plus the current segment we're appending. - let len = u32::try_from(len).unwrap(); - let init = if len > 0 { - Some(StaticMemoryInitializer { - offset, - data: module_data_size..module_data_size + len, - }) - } else { - None - }; - let idx = map.push(init).panic_on_oom(); + let idx = map.push(runtime_index.map(|i| (offset, i))).panic_on_oom(); assert_eq!(idx, memory); - module_data_size += len; + if let Some(runtime_index) = runtime_index { + new_initializers.push((idx, MemorySegmentOffset::Static(offset), runtime_index)); + } } self.data_align = Some(page_size); self.module.memory_initialization = MemoryInitialization::Static { map }; + self.memory_init = MemoryInit::Processed(new_initializers); + } + + /// Finalizes the initialization of tables. + /// + /// This is invoked after translation and notably uses + /// `Self::try_func_table_init` to attempt to optimize initialization of + /// tables into static precomputed images. + pub fn finalize_table_init(&mut self, tunables: &Tunables, types: &mut ModuleTypesBuilder) { + if tunables.table_lazy_init { + self.try_func_table_init(); + } + + // If any table has a non-null initializers, or if there's any active + // data segments, then a startup function is unconditionally required to + // configure the table. + if self + .table_initialization + .initial_values + .iter() + .any(|(_, v)| !matches!(v, TableInitialValue::Null)) + || !self.table_initialization.segments.is_empty() + { + self.require_startup_func(types); + } } /// Attempts to convert the module's table initializers to /// FuncTable form where possible. This enables lazy table /// initialization later by providing a one-to-one map of initial /// table values, without having to parse all segments. - pub fn try_func_table_init(&mut self) { + fn try_func_table_init(&mut self) { // This should be large enough to support very large Wasm // modules with huge funcref tables, but small enough to avoid // OOMs or DoS on truly sparse tables. @@ -1232,32 +1370,27 @@ impl ModuleTranslation<'_> { // First convert any element-initialized tables to images of just that // single function if the minimum size of the table allows doing so. - for ((_, init), (_, table)) in self - .module - .table_initialization - .initial_values - .iter_mut() - .zip( - self.module - .tables - .iter() - .skip(self.module.num_imported_tables), - ) - { + for ((i, init), (_, table)) in self.table_initialization.initial_values.iter_mut().zip( + self.module + .tables + .iter() + .skip(self.module.num_imported_tables), + ) { let table_size = table.limits.min; if table_size > MAX_FUNC_TABLE_SIZE { continue; } if let TableInitialValue::Expr(expr) = init { if let [ConstOp::RefFunc(f)] = expr.ops() { - *init = TableInitialValue::Null { - precomputed: try_vec![*f; table_size as usize].panic_on_oom(), - }; + assert!(self.module.table_initialization[i].is_empty()); + self.module.table_initialization[i] = + try_vec![*f; table_size as usize].panic_on_oom(); + *init = TableInitialValue::Null; } } } - let mut segments = mem::take(&mut self.module.table_initialization.segments) + let mut segments = mem::take(&mut self.table_initialization.segments) .into_iter() .peekable(); @@ -1326,18 +1459,18 @@ impl ModuleTranslation<'_> { TableSegmentElements::Expressions { .. } => break, }; - let precomputed = - match &mut self.module.table_initialization.initial_values[defined_index] { - TableInitialValue::Null { precomputed } => precomputed, + match &self.table_initialization.initial_values[defined_index] { + TableInitialValue::Null => {} - // If this table is still listed as an initial value here - // then that means the initial size of the table doesn't - // support a precomputed function list, so skip this. - // Technically this won't trap so it's possible to process - // further initializers, but that's left as a future - // optimization. - TableInitialValue::Expr(_) => break, - }; + // If this table is still listed as an initial value here + // then that means the initial size of the table doesn't + // support a precomputed function list, so skip this. + // Technically this won't trap so it's possible to process + // further initializers, but that's left as a future + // optimization. + TableInitialValue::Expr(_) => break, + } + let precomputed = &mut self.module.table_initialization[defined_index]; // At this point we're committing to pre-initializing the table // with the `segment` that's being iterated over. This segment is @@ -1355,6 +1488,27 @@ impl ModuleTranslation<'_> { // advance the iterator to see the next segment let _ = segments.next(); } - self.module.table_initialization.segments = segments.try_collect().panic_on_oom(); + self.table_initialization.segments = segments.try_collect().panic_on_oom(); + } + + /// Helper function to ratchet the `startup` function for this module as + /// `Always`. + fn require_startup_func(&mut self, types: &mut ModuleTypesBuilder) { + let ty = match self.module.startup { + ModuleStartup::None => types.startup_func_type().into(), + ModuleStartup::Always(_) => return, + ModuleStartup::IfMemoriesNeedInit(ty) => ty, + }; + self.module.startup = ModuleStartup::Always(ty); + } + + /// Helper function to ratchet the `startup` function for this module as + /// `IfMemoriesNeedInit`. + fn require_startup_func_if_memories_need_init(&mut self, types: &mut ModuleTypesBuilder) { + let ty = match self.module.startup { + ModuleStartup::None => types.startup_func_type().into(), + ModuleStartup::Always(_) | ModuleStartup::IfMemoriesNeedInit(_) => return, + }; + self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty); } } diff --git a/crates/environ/src/compile/module_types.rs b/crates/environ/src/compile/module_types.rs index a82843dc9ad7..082b3cab23f7 100644 --- a/crates/environ/src/compile/module_types.rs +++ b/crates/environ/src/compile/module_types.rs @@ -2,7 +2,7 @@ use crate::{ EngineOrModuleTypeIndex, EntityRef, ModuleInternedRecGroupIndex, ModuleInternedTypeIndex, ModuleTypes, PanicOnOom as _, TypeConvert, TypeIndex, WasmArrayType, WasmCompositeInnerType, WasmCompositeType, WasmExnType, WasmFieldType, WasmFuncType, WasmHeapType, WasmResult, - WasmStorageType, WasmStructType, WasmSubType, + WasmStorageType, WasmStructType, WasmSubType, WasmValType, collections::{TryClone as _, TryCow}, wasm_unsupported, }; @@ -59,6 +59,9 @@ pub struct ModuleTypesBuilder { /// If we are in the middle of defining a recursion group, this is the /// metadata about the recursion group we started defining. defining_rec_group: Option, + + /// Cache of the return value of [`Self::startup_func_type`]. + startup_func_type: Option, } impl ModuleTypesBuilder { @@ -72,6 +75,7 @@ impl ModuleTypesBuilder { wasmparser_to_wasmtime: HashMap::default(), already_seen: HashMap::default(), defining_rec_group: None, + startup_func_type: None, } } @@ -462,6 +466,45 @@ impl ModuleTypesBuilder { } Ok(composite_type.inner.unwrap_func()) } + + /// Gets a type for the "module startup" function, or `[] -> []` in the wasm + /// type system. + pub fn startup_func_type(&mut self) -> ModuleInternedTypeIndex { + *self.startup_func_type.get_or_insert_with(|| { + let idx = self.types.push(WasmSubType { + is_final: true, + supertype: None, + composite_type: WasmCompositeType { + inner: WasmCompositeInnerType::Func(WasmFuncType::new([], []).panic_on_oom()), + shared: false, + }, + }); + let next = self.types.next_ty(); + self.types.push_rec_group(idx..next); + idx + }) + } + + /// Smaller helper method to find a `ModuleInternedTypeIndex` which + /// corresponds to the `resource.drop` intrinsic in components, namely a + /// core wasm function type which takes one `i32` argument and has no + /// results. + /// + /// This is a bit of a hack right now as ideally this find operation + /// wouldn't be needed and instead the `ModuleInternedTypeIndex` itself + /// would be threaded through appropriately, but that's left for a future + /// refactoring. Try not to lean too hard on this method though. + pub fn find_resource_drop_signature(&self) -> Option { + self.wasm_types() + .find(|(_, ty)| { + ty.as_func().map_or(false, |sig| { + sig.params().len() == 1 + && sig.results().len() == 0 + && sig.params()[0] == WasmValType::I32 + }) + }) + .map(|(i, _)| i) + } } // Forward the indexing impl to the internal `ModuleTypes` diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs index 9e98fcac64a0..2bfc28a36145 100644 --- a/crates/environ/src/component/compiler.rs +++ b/crates/environ/src/component/compiler.rs @@ -10,7 +10,7 @@ pub trait ComponentCompiler: Send + Sync { /// Each trampoline is a member of the `Trampoline` enumeration and has a /// unique purpose and is translated differently. See the implementation of /// this trait for Cranelift for more information. - fn compile_trampoline( + fn compile_component_trampoline( &self, component: &ComponentTranslation, types: &ComponentTypesBuilder, diff --git a/crates/environ/src/component/types_builder.rs b/crates/environ/src/component/types_builder.rs index 3f8f7d94957d..d6cd8ec85bec 100644 --- a/crates/environ/src/component/types_builder.rs +++ b/crates/environ/src/component/types_builder.rs @@ -2,8 +2,8 @@ use crate::component::*; use crate::error::{Result, bail}; use crate::prelude::*; use crate::{ - EngineOrModuleTypeIndex, EntityType, ModuleInternedTypeIndex, ModuleTypes, ModuleTypesBuilder, - PrimaryMap, TypeConvert, WasmHeapType, WasmValType, + EngineOrModuleTypeIndex, EntityType, ModuleTypes, ModuleTypesBuilder, PrimaryMap, TypeConvert, + WasmHeapType, }; use cranelift_entity::EntityRef; use std::collections::HashMap; @@ -163,28 +163,6 @@ impl ComponentTypesBuilder { (self.component_types, ty) } - /// Smaller helper method to find a `ModuleInternedTypeIndex` which - /// corresponds to the `resource.drop` intrinsic in components, namely a - /// core wasm function type which takes one `i32` argument and has no - /// results. - /// - /// This is a bit of a hack right now as ideally this find operation - /// wouldn't be needed and instead the `ModuleInternedTypeIndex` itself - /// would be threaded through appropriately, but that's left for a future - /// refactoring. Try not to lean too hard on this method though. - pub fn find_resource_drop_signature(&self) -> Option { - self.module_types - .wasm_types() - .find(|(_, ty)| { - ty.as_func().map_or(false, |sig| { - sig.params().len() == 1 - && sig.results().len() == 0 - && sig.params()[0] == WasmValType::I32 - }) - }) - .map(|(i, _)| i) - } - /// Returns the underlying builder used to build up core wasm module types. /// /// Note that this is shared across all modules found within a component to diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rs index 8f0fd0e794ed..66bc7c189a8d 100644 --- a/crates/environ/src/key.rs +++ b/crates/environ/src/key.rs @@ -46,6 +46,10 @@ pub enum FuncKeyKind { /// A Wasmtime unsafe intrinsic function. #[cfg(feature = "component-model")] UnsafeIntrinsic = FuncKey::new_kind(0b1000), + + /// Initialization function for a module, such as initializing "complicated" + /// globals and passive element segments. + ModuleStartup = FuncKey::new_kind(0b1001), } impl From for u32 { @@ -73,6 +77,7 @@ impl FuncKeyKind { Self::PatchableToBuiltinTrampoline } x if x == Self::PulleyHostCall.into() => Self::PulleyHostCall, + x if x == Self::ModuleStartup.into() => Self::ModuleStartup, #[cfg(feature = "component-model")] x if x == Self::ComponentTrampoline.into() => Self::ComponentTrampoline, @@ -125,7 +130,9 @@ impl FuncKeyNamespace { /// Panics when given invalid raw representations. pub fn from_raw(raw: u32) -> Self { match FuncKeyKind::from_raw(raw & FuncKey::KIND_MASK) { - FuncKeyKind::DefinedWasmFunction | FuncKeyKind::ArrayToWasmTrampoline => Self(raw), + FuncKeyKind::DefinedWasmFunction + | FuncKeyKind::ArrayToWasmTrampoline + | FuncKeyKind::ModuleStartup => Self(raw), FuncKeyKind::WasmToArrayTrampoline | FuncKeyKind::WasmToBuiltinTrampoline | FuncKeyKind::PatchableToBuiltinTrampoline @@ -205,7 +212,6 @@ pub enum Abi { Patchable = 2, } -#[cfg(feature = "component-model")] impl Abi { fn from_raw(raw: u32) -> Self { match raw { @@ -254,6 +260,13 @@ pub enum FuncKey { /// A Wasmtime intrinsic function. #[cfg(feature = "component-model")] UnsafeIntrinsic(Abi, component::UnsafeIntrinsic), + + /// Initialization function for a module, such as initializing "complicated" + /// globals and passive element segments. + /// + /// This function has the `Abi` specified and will initialize the module + /// specified. + ModuleStartup(Abi, StaticModuleIndex), } impl Ord for FuncKey { @@ -342,6 +355,13 @@ impl FuncKey { let index = intrinsic.index(); (namespace, index) } + + FuncKey::ModuleStartup(abi, module) => { + assert_eq!(module.as_u32() & Self::KIND_MASK, 0); + let namespace = FuncKeyKind::ModuleStartup.into_raw() | module.as_u32(); + let index = abi.into_raw(); + (namespace, index) + } }; (FuncKeyNamespace(namespace), FuncKeyIndex(index)) } @@ -376,6 +396,7 @@ impl FuncKey { FuncKey::ResourceDropTrampoline => Abi::Wasm, #[cfg(feature = "component-model")] FuncKey::UnsafeIntrinsic(abi, _) => abi, + FuncKey::ModuleStartup(abi, _) => abi, } } @@ -462,6 +483,12 @@ impl FuncKey { let intrinsic = component::UnsafeIntrinsic::from_u32(b); Self::UnsafeIntrinsic(abi, intrinsic) } + + FuncKeyKind::ModuleStartup => { + let module = StaticModuleIndex::from_u32(a & Self::MODULE_MASK); + let abi = Abi::from_raw(b); + Self::ModuleStartup(abi, module) + } } } @@ -556,7 +583,9 @@ impl FuncKey { /// looking up the StoreCode (or having access to the Store). pub fn is_store_invariant(&self) -> bool { match self { - Self::DefinedWasmFunction(..) | Self::ArrayToWasmTrampoline(..) => false, + Self::DefinedWasmFunction(..) + | Self::ArrayToWasmTrampoline(..) + | Self::ModuleStartup(..) => false, Self::WasmToArrayTrampoline(..) | Self::WasmToBuiltinTrampoline(..) | Self::PatchableToBuiltinTrampoline(..) diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 1983b9729b74..45e228449bb6 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -7,8 +7,8 @@ use cranelift_entity::{EntityRef, packed_option::ReservedValue}; use serde_derive::{Deserialize, Serialize}; /// A WebAssembly linear memory initializer. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MemoryInitializer { +#[derive(Clone, Debug)] +pub struct MemoryInitializer<'a> { /// The index of a linear memory to initialize. pub memory_index: MemoryIndex, /// The base offset to start this segment at. @@ -17,19 +17,7 @@ pub struct MemoryInitializer { /// /// This range indexes into a separately stored data section which will be /// provided with the compiled module's code as well. - pub data: Range, -} - -/// Similar to the above `MemoryInitializer` but only used when memory -/// initializers are statically known to be valid. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct StaticMemoryInitializer { - /// The 64-bit offset, in bytes, of where this initializer starts. - pub offset: u64, - - /// The range of data to write at `offset`, where these indices are indexes - /// into the compiled wasm module's data section. - pub data: Range, + pub data: &'a [u8], } /// The type of WebAssembly linear memory initialization to use for a module. @@ -47,7 +35,7 @@ pub enum MemoryInitialization { /// data segments when the module is instantiated. /// /// This is the default memory initialization type. - Segmented(TryVec), + Segmented, /// Memory initialization is statically known and involves a single `memcpy` /// or otherwise simply making the defined data visible. @@ -81,13 +69,13 @@ pub enum MemoryInitialization { /// /// The offset, range base, and range end are all guaranteed to be page /// aligned to the page size passed in to `try_static_init`. - map: TryPrimaryMap>, + map: TryPrimaryMap>, }, } impl Default for MemoryInitialization { fn default() -> Self { - Self::Segmented(TryVec::new()) + Self::Segmented } } @@ -96,121 +84,10 @@ impl MemoryInitialization { /// `MemoryInitialization::Segmented`. pub fn is_segmented(&self) -> bool { match self { - MemoryInitialization::Segmented(_) => true, + MemoryInitialization::Segmented => true, _ => false, } } - - /// Performs the memory initialization steps for this set of initializers. - /// - /// This will perform wasm initialization in compliance with the wasm spec - /// and how data segments are processed. This doesn't need to necessarily - /// only be called as part of initialization, however, as it's structured to - /// allow learning about memory ahead-of-time at compile time possibly. - /// - /// This function will return true if all memory initializers are processed - /// successfully. If any initializer hits an error or, for example, a - /// global value is needed but `None` is returned, then false will be - /// returned. At compile-time this typically means that the "error" in - /// question needs to be deferred to runtime, and at runtime this means - /// that an invalid initializer has been found and a trap should be - /// generated. - pub fn init_memory(&self, state: &mut dyn InitMemory) -> bool { - let initializers = match self { - // Fall through below to the segmented memory one-by-one - // initialization. - MemoryInitialization::Segmented(list) => list, - - // If previously switched to static initialization then pass through - // all those parameters here to the `write` callback. - // - // Note that existence of `Static` already guarantees that all - // indices are in-bounds. - MemoryInitialization::Static { map } => { - for (index, init) in map { - if let Some(init) = init { - let result = state.write(index, init); - if !result { - return result; - } - } - } - return true; - } - }; - - for initializer in initializers { - let &MemoryInitializer { - memory_index, - ref offset, - ref data, - } = initializer; - - // First up determine the start/end range and verify that they're - // in-bounds for the initial size of the memory at `memory_index`. - // Note that this can bail if we don't have access to globals yet - // (e.g. this is a task happening before instantiation at - // compile-time). - let start = match state.eval_offset(memory_index, offset) { - Some(start) => start, - None => return false, - }; - let len = u64::try_from(data.len()).unwrap(); - let end = match start.checked_add(len) { - Some(end) => end, - None => return false, - }; - - match state.memory_size_in_bytes(memory_index) { - Ok(max) => { - if end > max { - return false; - } - } - - // Note that computing the minimum can overflow if the page size - // is the default 64KiB and the memory's minimum size in pages - // is `1 << 48`, the maximum number of minimum pages for 64-bit - // memories. We don't return `false` to signal an error here and - // instead defer the error to runtime, when it will be - // impossible to allocate that much memory anyways. - Err(_) => {} - } - - // The limits of the data segment have been validated at this point - // so the `write` callback is called with the range of data being - // written. Any erroneous result is propagated upwards. - let init = StaticMemoryInitializer { - offset: start, - data: data.clone(), - }; - let result = state.write(memory_index, &init); - if !result { - return result; - } - } - - return true; - } -} - -/// The various callbacks provided here are used to drive the smaller bits of -/// memory initialization. -pub trait InitMemory { - /// Returns the size, in bytes, of the memory specified. For compile-time - /// purposes this would be the memory type's minimum size. - fn memory_size_in_bytes(&mut self, memory_index: MemoryIndex) -> Result; - - /// Returns the value of the constant expression, as a `u64`. Note that - /// this may involve zero-extending a 32-bit global to a 64-bit number. May - /// return `None` to indicate that the expression involves a value which is - /// not available yet. - fn eval_offset(&mut self, memory_index: MemoryIndex, expr: &ConstExpr) -> Option; - - /// A callback used to actually write data. This indicates that the - /// specified memory must receive the specified range of data at the - /// specified offset. This can return false on failure. - fn write(&mut self, memory_index: MemoryIndex, init: &StaticMemoryInitializer) -> bool; } /// Table initialization data for all tables in the module. @@ -240,16 +117,7 @@ pub struct TableInitialization { pub enum TableInitialValue { /// Initialize each table element to null, optionally setting some elements /// to non-null given the precomputed image. - Null { - /// A precomputed image of table initializers for this table. - /// - /// This image is constructed during `try_func_table_init` and - /// null-initialized elements are represented with - /// `FuncIndex::reserved_value()`. Note that this image is empty by - /// default and may not encompass the entire span of the table in which - /// case the elements are initialized to null. - precomputed: TryVec, - }, + Null, /// An arbitrary const expression. Expr(ConstExpr), } @@ -285,6 +153,14 @@ pub enum TableSegmentElements { } impl TableSegmentElements { + /// Returns the type of this segment. + pub fn ty(&self) -> WasmRefType { + match self { + Self::Functions(_) => WasmRefType::FUNCREF, + Self::Expressions { ty, .. } => *ty, + } + } + /// Returns the number of elements in this segment. pub fn len(&self) -> u64 { match self { @@ -313,20 +189,38 @@ pub struct Module { /// Exported entities. pub exports: TryIndexMap, - /// The module "start" function, if present. - pub start_func: Option, + /// Whether or not this module has a start function, + pub startup: ModuleStartup, - /// WebAssembly table initialization data, per table. - pub table_initialization: TableInitialization, + /// Precompute per-table static images, if applicable. + /// + /// This map tracks, for each defined table in this module, the initial + /// precomputed contents of the table. This is only applicable for funcref + /// tables and the `TryVec` here uses `FuncIndex::reserved_value()` for null + /// entries. This structure is filled in if table initialization is detected + /// to be infallible as part of [`ModuleTranslation::finalize_table_init`]. + pub table_initialization: TryPrimaryMap>, /// WebAssembly linear memory initializer. + /// + /// This will track how memory is initialized, either exclusively via + /// segments or if some memories can be initialized with static images. This + /// is computed during [`ModuleTranslation::finalize_memory_init`]. pub memory_initialization: MemoryInitialization, /// WebAssembly passive elements. - pub passive_elements: TryPrimaryMap, + /// + /// This is a map of all passive element segments to their type and the + /// initial size of the segment. Note that the contents of the segment are + /// initialized by compiled code. + pub passive_elements: TryPrimaryMap, - /// Where passive data segments are located in the module's image. - pub passive_data: TryPrimaryMap>, + /// Where runtime data segments are located in the module's image. + /// + /// Note that this does not directly correspond to either active or passive + /// data segments. Those are massaged during + /// [`ModuleTranslation::finalize_memory_init`] into the form used here. + pub runtime_data: TryPrimaryMap>, /// Types declared in the wasm module. pub types: TryPrimaryMap, @@ -368,8 +262,18 @@ pub struct Module { /// WebAssembly global variables. pub globals: TryPrimaryMap, - /// WebAssembly global initializers for locally-defined globals. - pub global_initializers: TryPrimaryMap, + /// "Simple" WebAssembly global initializers for locally-defined globals. + /// + /// This map does not track initialization of all globals in this module, + /// but only those considered "simple" which can be easily evaluated at + /// compile-time. For example an initialization expression of `i32.const N` + /// is considered simple. These globals are manually initialized in the + /// host. + /// + /// This is all in contrast to [`ModuleTranslation::global_initializers`] + /// which is processed in compiled code and initialized after the instance + /// has been created. + pub global_initializers: TryVec<(DefinedGlobalIndex, GlobalConstValue)>, /// WebAssembly exception and control tags. pub tags: TryPrimaryMap, @@ -400,11 +304,11 @@ impl Module { name: Default::default(), initializers: Default::default(), exports: Default::default(), - start_func: Default::default(), + startup: ModuleStartup::None, table_initialization: Default::default(), memory_initialization: Default::default(), passive_elements: Default::default(), - passive_data: Default::default(), + runtime_data: Default::default(), types: Default::default(), num_imported_funcs: Default::default(), num_imported_tables: Default::default(), @@ -690,11 +594,11 @@ impl TypeTrace for Module { name: _, initializers: _, exports: _, - start_func: _, + startup, table_initialization: _, memory_initialization: _, passive_elements: _, - passive_data: _, + runtime_data: _, types, num_imported_funcs: _, num_imported_tables: _, @@ -726,6 +630,7 @@ impl TypeTrace for Module { for t in tags.values() { t.trace(func)?; } + startup.trace(func)?; Ok(()) } @@ -741,11 +646,11 @@ impl TypeTrace for Module { name: _, initializers: _, exports: _, - start_func: _, + startup, table_initialization: _, memory_initialization: _, passive_elements: _, - passive_data: _, + runtime_data: _, types, num_imported_funcs: _, num_imported_tables: _, @@ -777,10 +682,33 @@ impl TypeTrace for Module { for t in tags.values_mut() { t.trace_mut(func)?; } + startup.trace_mut(func)?; Ok(()) } } +impl TypeTrace for ModuleStartup { + fn trace(&self, func: &mut F) -> Result<(), E> + where + F: FnMut(EngineOrModuleTypeIndex) -> Result<(), E>, + { + match self { + ModuleStartup::None => Ok(()), + ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => func(*t), + } + } + + fn trace_mut(&mut self, func: &mut F) -> Result<(), E> + where + F: FnMut(&mut EngineOrModuleTypeIndex) -> Result<(), E>, + { + match self { + ModuleStartup::None => Ok(()), + ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => func(t), + } + } +} + /// Type information about functions in a wasm module. #[derive(Debug, Serialize, Deserialize)] pub struct FunctionType { @@ -821,3 +749,34 @@ impl FunctionType { #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] pub struct FuncRefIndex(u32); cranelift_entity::entity_impl!(FuncRefIndex); + +/// Different means of startup for a wasm module. +#[derive(Debug, Serialize, Deserialize)] +pub enum ModuleStartup { + /// No startup is necessary. + None, + + /// Startup is always required, for example to apply active table segments. + /// + /// The type of the startup function, of wasm signature `[] -> []`, is + /// provided here. + Always(EngineOrModuleTypeIndex), + + /// Startup is only required if some linear memory within this module, at + /// runtime, says `needs_init() == true`. + /// + /// This special mode of startup indicates that the startup function has no + /// purpose other than to initialize the initial contents of + /// `MemoryInitialization::Static` linear memories. In this situation if all + /// memories say `needs_init() == false` then the startup function won't + /// actually do anything meaning that it can be optimized slightly by + /// skipping it entirely. + IfMemoriesNeedInit(EngineOrModuleTypeIndex), +} + +impl ModuleStartup { + /// Returns if this is `ModuleStartup::None`. + pub fn is_none(&self) -> bool { + matches!(self, Self::None) + } +} diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs index 0f78e765cac5..c3173b3819e5 100644 --- a/crates/environ/src/module_artifacts.rs +++ b/crates/environ/src/module_artifacts.rs @@ -552,7 +552,8 @@ impl CompiledFunctionsTable { FuncKeyKind::ArrayToWasmTrampoline | FuncKeyKind::WasmToBuiltinTrampoline - | FuncKeyKind::PatchableToBuiltinTrampoline => false, + | FuncKeyKind::PatchableToBuiltinTrampoline + | FuncKeyKind::ModuleStartup => false, #[cfg(feature = "component-model")] FuncKeyKind::ComponentTrampoline @@ -569,7 +570,8 @@ impl CompiledFunctionsTable { | FuncKeyKind::WasmToArrayTrampoline | FuncKeyKind::WasmToBuiltinTrampoline | FuncKeyKind::PatchableToBuiltinTrampoline - | FuncKeyKind::PulleyHostCall => false, + | FuncKeyKind::PulleyHostCall + | FuncKeyKind::ModuleStartup => false, #[cfg(feature = "component-model")] FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rs index 0ee06f9378a3..9841f45ebcc2 100644 --- a/crates/environ/src/types.rs +++ b/crates/environ/src/types.rs @@ -1699,13 +1699,19 @@ impl Default for VMSharedTypeIndex { pub struct DataIndex(u32); entity_impl_with_try_clone!(DataIndex); -/// Index type of a passive data segment inside the WebAssembly module. +/// Index into data segments needed at runtime by a module. /// -/// Not a spec-level concept, just used to get dense index spaces for passive -/// data segments inside of Wasmtime. +/// This does not directly correspond to either active or passive data segments +/// in the wasm spec. Instead this is a concept purely for Wasmtime and +/// organizing memory initialization within the +/// `ModuleTranslation::finalize_memory_init` function, for example. +/// +/// Passive data segments at runtime all have a corresponding +/// `RuntimeDataIndex`, but active data segments maybe coalesced or mutated if +/// they're statically evaluated. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] -pub struct PassiveDataIndex(u32); -entity_impl_with_try_clone!(PassiveDataIndex); +pub struct RuntimeDataIndex(u32); +entity_impl_with_try_clone!(RuntimeDataIndex); /// Index type of an element segment inside the WebAssembly module. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] @@ -2007,7 +2013,7 @@ impl ConstExpr { /// A global's constant value, known at compile time. #[expect(missing_docs, reason = "self-describing variants")] -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub enum GlobalConstValue { I32(i32), I64(i64), @@ -2027,7 +2033,7 @@ pub enum ConstOp { V128Const(u128), GlobalGet(GlobalIndex), RefI31, - RefNull(WasmHeapTopType), + RefNull(WasmHeapType), RefFunc(FuncIndex), I32Add, I32Sub, @@ -2069,7 +2075,7 @@ impl ConstOp { O::F32Const { value } => Self::F32Const(value.bits()), O::F64Const { value } => Self::F64Const(value.bits()), O::V128Const { value } => Self::V128Const(u128::from_le_bytes(*value.bytes())), - O::RefNull { hty } => Self::RefNull(env.convert_heap_type(hty)?.top()), + O::RefNull { hty } => Self::RefNull(env.convert_heap_type(hty)?), O::RefFunc { function_index } => Self::RefFunc(FuncIndex::from_u32(function_index)), O::GlobalGet { global_index } => Self::GlobalGet(GlobalIndex::from_u32(global_index)), O::RefI31 => Self::RefI31, diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 74c50dd201e9..3437c1467426 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -31,13 +31,14 @@ // globals: [VMGlobalDefinition; module.num_defined_globals], // tags: [VMTagDefinition; module.num_defined_tags], // func_refs: [VMFuncRef; module.num_escaped_funcs], -// passive_data_bases: [*const u8; module.num_passive_data], -// passive_data_lengths: [u32; module.num_passive_data], +// startup_func_ref: [VMFuncRef; module.has_startup_func ? 1 : 0], +// runtime_data_bases: [*const u8; module.num_runtime_data], +// runtime_data_lengths: [u32; module.num_runtime_data], // } use crate::{ DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex, - FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, PassiveDataIndex, TableIndex, + FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex, TagIndex, }; use cranelift_entity::packed_option::ReservedValue; @@ -90,8 +91,10 @@ pub struct VMOffsets

{ /// The number of escaped functions in the module, the size of the func_refs /// array. pub num_escaped_funcs: u32, - /// The number of passive data segments in the module. - pub num_passive_data: u32, + /// The number of runtime data segments in the module. + pub num_runtime_data: u32, + /// Whether or not the module has a start function. + pub has_startup_func: bool, // precalculated offsets of various member fields imported_functions: u32, @@ -105,8 +108,9 @@ pub struct VMOffsets

{ defined_globals: u32, defined_tags: u32, defined_func_refs: u32, - passive_data_bases: u32, - passive_data_lengths: u32, + startup_func_ref: u32, + runtime_data_bases: u32, + runtime_data_lengths: u32, size: u32, } @@ -607,8 +611,10 @@ pub struct VMOffsetsFields

{ /// The number of escaped functions in the module, the size of the function /// references array. pub num_escaped_funcs: u32, - /// The number of passive data segments in the module. - pub num_passive_data: u32, + /// The number of runtime data segments in the module. + pub num_runtime_data: u32, + /// Whether or not the module has a start function. + pub has_startup_func: bool, } impl VMOffsets

{ @@ -635,7 +641,8 @@ impl VMOffsets

{ num_defined_globals: cast_to_u32(module.globals.len() - module.num_imported_globals), num_defined_tags: cast_to_u32(module.tags.len() - module.num_imported_tags), num_escaped_funcs: cast_to_u32(module.num_escaped_funcs), - num_passive_data: cast_to_u32(module.passive_data.len()), + num_runtime_data: cast_to_u32(module.runtime_data.len()), + has_startup_func: !module.startup.is_none(), }) } @@ -667,7 +674,8 @@ impl VMOffsets

{ num_defined_tags: _, num_owned_memories: _, num_escaped_funcs: _, - num_passive_data: _, + num_runtime_data: _, + has_startup_func: _, // used as the initial size below size, @@ -696,8 +704,9 @@ impl VMOffsets

{ } calculate_sizes! { - passive_data_lengths: "passive data lengths", - passive_data_bases: "passive data base pointers", + runtime_data_lengths: "runtime data lengths", + runtime_data_bases: "runtime data base pointers", + startup_func_ref: "startup funcref", defined_func_refs: "module functions", defined_tags: "defined tags", defined_globals: "defined globals", @@ -728,7 +737,8 @@ impl From> for VMOffsets

{ num_defined_globals: fields.num_defined_globals, num_defined_tags: fields.num_defined_tags, num_escaped_funcs: fields.num_escaped_funcs, - num_passive_data: fields.num_passive_data, + num_runtime_data: fields.num_runtime_data, + has_startup_func: fields.has_startup_func, imported_functions: 0, imported_tables: 0, imported_memories: 0, @@ -740,8 +750,9 @@ impl From> for VMOffsets

{ defined_globals: 0, defined_tags: 0, defined_func_refs: 0, - passive_data_bases: 0, - passive_data_lengths: 0, + startup_func_ref: 0, + runtime_data_bases: 0, + runtime_data_lengths: 0, size: 0, }; @@ -800,8 +811,13 @@ impl From> for VMOffsets

{ ret.num_escaped_funcs, ret.ptr.size_of_vm_func_ref(), ), - size(passive_data_bases) = cmul(ret.num_passive_data, ret.ptr.size()), - size(passive_data_lengths) = cmul(ret.num_passive_data, 4), + size(startup_func_ref) = if ret.has_startup_func { + ret.ptr.size_of_vm_func_ref() + } else { + 0 + }, + size(runtime_data_bases) = cmul(ret.num_runtime_data, ret.ptr.size()), + size(runtime_data_lengths) = cmul(ret.num_runtime_data, 4), } ret.size = next_field_offset; @@ -1050,16 +1066,16 @@ impl VMOffsets

{ self.defined_func_refs } - /// The offset of the `passive_data_bases` array. + /// The offset of the `runtime_data_bases` array. #[inline] - pub fn vmctx_passive_data_bases_begin(&self) -> u32 { - self.passive_data_bases + pub fn vmctx_runtime_data_bases_begin(&self) -> u32 { + self.runtime_data_bases } - /// The offset of the `passive_data_lengths` array. + /// The offset of the `runtime_data_lengths` array. #[inline] - pub fn vmctx_passive_data_lengths_begin(&self) -> u32 { - self.passive_data_lengths + pub fn vmctx_runtime_data_lengths_begin(&self) -> u32 { + self.runtime_data_lengths } /// Return the size of the `VMContext` allocation. @@ -1154,20 +1170,29 @@ impl VMOffsets

{ self.vmctx_func_refs_begin() + index.as_u32() * u32::from(self.ptr.size_of_vm_func_ref()) } - /// Return the offset to the base of the passive data segment at `index`. + /// Returns the offset to the `VMFuncRef` for the module startup function. + /// + /// Panics if this module does not have a startup function. + #[inline] + pub fn vmctx_startup_func_ref(&self) -> u32 { + assert!(self.has_startup_func); + self.startup_func_ref + } + + /// Return the offset to the base of the runtime data segment at `index`. #[inline] - pub fn vmctx_passive_data_base(&self, index: PassiveDataIndex) -> u32 { + pub fn vmctx_runtime_data_base(&self, index: RuntimeDataIndex) -> u32 { assert!(!index.is_reserved_value()); - assert!(index.as_u32() < self.num_passive_data); - self.vmctx_passive_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size()) + assert!(index.as_u32() < self.num_runtime_data); + self.vmctx_runtime_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size()) } - /// Return the offset to the length of the passive data segment at `index`. + /// Return the offset to the length of the runtime data segment at `index`. #[inline] - pub fn vmctx_passive_data_length(&self, index: PassiveDataIndex) -> u32 { + pub fn vmctx_runtime_data_length(&self, index: RuntimeDataIndex) -> u32 { assert!(!index.is_reserved_value()); - assert!(index.as_u32() < self.num_passive_data); - self.vmctx_passive_data_lengths_begin() + index.as_u32() * 4 + assert!(index.as_u32() < self.num_runtime_data); + self.vmctx_runtime_data_lengths_begin() + index.as_u32() * 4 } /// Return the offset to the `wasm_call` field in `*const VMFunctionBody` index `index`. diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs index 3696f325ab30..2994644c5d75 100644 --- a/crates/wasmtime/src/compile.rs +++ b/crates/wasmtime/src/compile.rs @@ -85,10 +85,11 @@ pub(crate) fn build_module_artifacts( ) .translate(parser, wasm) .context("failed to parse WebAssembly module")?; + prepare_translation(engine, compiler, &mut translation, &mut types); let functions = mem::take(&mut translation.function_body_inputs); let compile_inputs = CompileInputs::for_module(&types, &translation, functions); - let unlinked_compile_outputs = compile_inputs.compile(engine)?; + let unlinked_compile_outputs = compile_inputs.compile(engine, &types)?; let PreLinkOutput { needs_gc_heap, compiled_funcs, @@ -165,6 +166,15 @@ pub(crate) fn build_component_artifacts( .translate(binary) .context("failed to parse WebAssembly module")?; + for (_, translation) in module_translations.iter_mut() { + prepare_translation( + engine, + compiler, + translation, + types.module_types_builder_mut(), + ); + } + let compile_inputs = CompileInputs::for_component( engine, &types, @@ -174,7 +184,7 @@ pub(crate) fn build_component_artifacts( (i, &*translation, functions) }), ); - let unlinked_compile_outputs = compile_inputs.compile(&engine)?; + let unlinked_compile_outputs = compile_inputs.compile(&engine, types.module_types_builder())?; let PreLinkOutput { needs_gc_heap, @@ -230,6 +240,26 @@ pub(crate) fn build_component_artifacts( Ok((result, Some(artifacts))) } +fn prepare_translation( + engine: &Engine, + compiler: &dyn Compiler, + translation: &mut ModuleTranslation<'_>, + types: &mut ModuleTypesBuilder, +) { + // If configured attempt to use static memory initialization + // which can either at runtime be implemented as a single memcpy + // to initialize memory or otherwise enabling + // virtual-memory-tricks such as mmap'ing from a file to get + // copy-on-write. + let align = compiler.page_size_align(); + let max_always_allowed = engine.config().memory_guaranteed_dense_image_size; + translation.finalize_memory_init(engine.tunables(), align, max_always_allowed, types); + + // Attempt to convert table initializer segments to FuncTable + // representation where possible, to enable table lazy init. + translation.finalize_table_init(engine.tunables(), types); +} + type CompileInput<'a> = Box Result> + Send + 'a>; struct CompileOutput<'a> { @@ -357,7 +387,7 @@ impl<'a> CompileInputs<'a> { ); let function = compiler .component_compiler() - .compile_trampoline(component, types, key, abi, tunables, &symbol) + .compile_component_trampoline(component, types, key, abi, tunables, &symbol) .with_context(|| format!("failed to compile {symbol}"))?; Ok(CompileOutput { key, @@ -378,12 +408,16 @@ impl<'a> CompileInputs<'a> { // resources from the embedder and otherwise won't be explicitly // requested through initializers above or such. if component.component.num_resources > 0 { - if let Some(sig) = types.find_resource_drop_signature() { + if types + .module_types_builder() + .find_resource_drop_signature() + .is_some() + { ret.push_input(move |compiler| { let key = FuncKey::ResourceDropTrampoline; let symbol = "resource_drop_trampoline".to_string(); let function = compiler - .compile_wasm_to_array_trampoline(types[sig].unwrap_func(), key, &symbol) + .compile_trampoline(None, key, types.module_types_builder(), &symbol) .with_context(|| format!("failed to compile `{symbol}`"))?; Ok(CompileOutput { key, @@ -493,7 +527,7 @@ impl<'a> CompileInputs<'a> { func_index.as_u32() ); let function = compiler - .compile_array_to_wasm_trampoline(translation, types, key, &symbol) + .compile_trampoline(Some(translation), key, types, &symbol) .with_context(|| format!("failed to compile: {symbol}"))?; Ok(CompileOutput { key, @@ -506,6 +540,26 @@ impl<'a> CompileInputs<'a> { }); } } + + if !translation.module.startup.is_none() { + for abi in [Abi::Wasm, Abi::Array] { + self.push_input(move |compiler| { + let key = FuncKey::ModuleStartup(abi, module); + let symbol = format!("module_start[{}]::{abi:?}", module.as_u32()); + let function = compiler + .compile_trampoline(Some(translation), key, types, &symbol) + .with_context(|| format!("failed to compile: {symbol}"))?; + Ok(CompileOutput { + key, + function, + symbol, + start_srcloc: FilePos::default(), + translation: None, + func_body: None, + }) + }); + } + } } let mut trampoline_types_seen = HashSet::new(); @@ -514,7 +568,6 @@ impl<'a> CompileInputs<'a> { if !is_new { continue; } - let trampoline_func_ty = types[trampoline_type_index].unwrap_func(); self.push_input(move |compiler| { let key = FuncKey::WasmToArrayTrampoline(trampoline_type_index); let symbol = format!( @@ -522,7 +575,7 @@ impl<'a> CompileInputs<'a> { trampoline_type_index.as_u32() ); let function = compiler - .compile_wasm_to_array_trampoline(trampoline_func_ty, key, &symbol) + .compile_trampoline(None, key, types, &symbol) .with_context(|| format!("failed to compile: {symbol}"))?; Ok(CompileOutput { key, @@ -538,7 +591,11 @@ impl<'a> CompileInputs<'a> { /// Compile these `CompileInput`s (maybe in parallel) and return the /// resulting `UnlinkedCompileOutput`s. - fn compile(self, engine: &Engine) -> Result> { + fn compile( + self, + engine: &Engine, + types: &'a ModuleTypesBuilder, + ) -> Result> { let compiler = engine.try_compiler()?; if self.inputs.len() > 0 && cfg!(miri) { @@ -597,7 +654,7 @@ the use case. // wasmtime-builtin functions are necessary. If so those need to be // collected and then those trampolines additionally need to be // compiled. - compile_required_builtins(engine, &mut raw_outputs)?; + compile_required_builtins(engine, types, &mut raw_outputs)?; // Bucket the outputs by kind. let mut outputs: BTreeMap = BTreeMap::new(); @@ -910,7 +967,8 @@ fn is_inlining_function(key: FuncKey) -> bool { FuncKey::ArrayToWasmTrampoline(..) | FuncKey::WasmToArrayTrampoline(..) | FuncKey::WasmToBuiltinTrampoline(..) - | FuncKey::PatchableToBuiltinTrampoline(..) => false, + | FuncKey::PatchableToBuiltinTrampoline(..) + | FuncKey::ModuleStartup(..) => false, #[cfg(feature = "component-model")] FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false, @@ -934,7 +992,11 @@ fn inlining_functions<'a>( }) } -fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec) -> Result<()> { +fn compile_required_builtins<'a>( + engine: &Engine, + types: &'a ModuleTypesBuilder, + raw_outputs: &mut Vec>, +) -> Result<()> { let compiler = engine.try_compiler()?; let mut builtins = HashSet::new(); let mut new_inputs: Vec> = Vec::new(); @@ -951,7 +1013,7 @@ fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec unreachable!(), }; let mut function = compiler - .compile_wasm_to_builtin(key, &symbol) + .compile_trampoline(None, key, types, &symbol) .with_context(|| format!("failed to compile `{symbol}`"))?; if let Some(compiler) = compiler.inlining_compiler() { compiler.finish_compiling(&mut function, None, &symbol)?; @@ -1116,26 +1178,7 @@ impl FunctionIndices { let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables); let modules = translations .into_iter() - .map(|(_, mut translation)| { - // If configured attempt to use static memory initialization - // which can either at runtime be implemented as a single memcpy - // to initialize memory or otherwise enabling - // virtual-memory-tricks such as mmap'ing from a file to get - // copy-on-write. - if engine.tunables().memory_init_cow { - let align = compiler.page_size_align(); - let max_always_allowed = engine.config().memory_guaranteed_dense_image_size; - translation.try_static_init(align, max_always_allowed); - } - - // Attempt to convert table initializer segments to FuncTable - // representation where possible, to enable table lazy init. - if engine.tunables().table_lazy_init { - translation.try_func_table_init(); - } - - obj.append(translation) - }) + .map(|(_, translation)| obj.append(translation)) .collect::>>()?; let artifacts = Artifacts { diff --git a/crates/wasmtime/src/runtime/code.rs b/crates/wasmtime/src/runtime/code.rs index 3d6af317c3fb..1faf0ac073fa 100644 --- a/crates/wasmtime/src/runtime/code.rs +++ b/crates/wasmtime/src/runtime/code.rs @@ -9,11 +9,9 @@ use alloc::boxed::Box; use alloc::sync::Arc; use core::ops::{Add, Range, Sub}; use wasmtime_core::error::OutOfMemory; -use wasmtime_environ::DefinedFuncIndex; -use wasmtime_environ::ModuleTypes; -use wasmtime_environ::StaticModuleIndex; #[cfg(feature = "component-model")] use wasmtime_environ::component::ComponentTypes; +use wasmtime_environ::{FuncKey, ModuleTypes, StaticModuleIndex}; macro_rules! define_pc_kind { ($ty:ident) => { @@ -431,30 +429,10 @@ impl<'a> ModuleWithCode<'a> { } /// Returns the slice in the text section of the function that - /// `index` points to. + /// `key` points to. #[inline] - pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] { - let range = self - .module - .compiled_module() - .finished_function_range(def_func_index); + pub fn function(&self, key: FuncKey) -> &[u8] { + let range = self.module.compiled_module().function_range(key); &self.store_code.text()[range] } - - /// Get the array-to-Wasm trampoline for the function `index` - /// points to, as a slice of raw code that can be converted to a - /// callable function pointer. - /// - /// If the function `index` points to does not escape, then `None` is - /// returned. - /// - /// These trampolines are used for array callers (e.g. `Func::new`) - /// calling Wasm callees. - pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> { - let range = self - .module - .compiled_module() - .array_to_wasm_trampoline_range(def_func_index)?; - Some(&self.store_code.text()[range]) - } } diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rs index 489caf0a8e3d..7b4f66cee674 100644 --- a/crates/wasmtime/src/runtime/component/component.rs +++ b/crates/wasmtime/src/runtime/component/component.rs @@ -967,6 +967,7 @@ mod tests { use wasmtime_environ::MemoryInitialization; #[test] + #[cfg_attr(miri, ignore)] fn cow_on_by_default() { let mut config = Config::new(); config.wasm_component_model(true); diff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs index d03ba277f12b..eb2f392270ac 100644 --- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs +++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs @@ -7,7 +7,7 @@ use core::convert::Infallible; use core::fmt::{self, Debug}; use core::hash::{Hash, Hasher}; use core::marker; -use core::ops::{Deref, DerefMut}; +use core::ops::Deref; mod sealed { use super::*; @@ -209,27 +209,3 @@ impl RootedGcRefImpl for OwnedRooted { match self.inner {} } } - -pub(crate) struct OpaqueRootScope { - store: S, -} - -impl Deref for OpaqueRootScope { - type Target = S; - - fn deref(&self) -> &Self::Target { - &self.store - } -} - -impl DerefMut for OpaqueRootScope { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.store - } -} - -impl OpaqueRootScope { - pub(crate) fn new(store: S) -> Self { - OpaqueRootScope { store } - } -} diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs index f2133cc2759e..1b4a53971020 100644 --- a/crates/wasmtime/src/runtime/instance.rs +++ b/crates/wasmtime/src/runtime/instance.rs @@ -14,7 +14,6 @@ use crate::{ }; use alloc::sync::Arc; use core::ptr::NonNull; -use wasmparser::WasmFeatures; use wasmtime_environ::{ EntityIndex, EntityType, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TagIndex, TypeTrace, }; @@ -255,21 +254,26 @@ impl Instance { imports: Imports<'_>, asyncness: Asyncness, ) -> Result { - let (instance, start) = { + let instance = { let (mut limiter, store) = store.0.resource_limiter_and_store_opaque(); // SAFETY: the safety contract of `new_raw` is the same as this // function. - unsafe { Instance::new_raw(store, limiter.as_mut(), module, imports, asyncness).await? } + unsafe { Instance::new_raw(store, limiter.as_mut(), module, imports).await? } }; - if let Some(start) = start { + + // If this instance requires startup, which is a dynamic decision made + // at this point in conjunction with analysis at compile time, the + // instance gets started. Note that this isn't just the wasm start + // function itself, but it's finalization of initialization of this + // instance, for example for complicated global initialization + // expressions. + if instance.id.get_mut(store.0).needs_startup() { if asyncness == Asyncness::No { - instance.start_raw(store, start)?; + instance.start_raw(store)?; } else { #[cfg(feature = "async")] { - store - .on_fiber(|store| instance.start_raw(store, start)) - .await??; + store.on_fiber(|store| instance.start_raw(store)).await??; } #[cfg(not(feature = "async"))] unreachable!(); @@ -298,8 +302,7 @@ impl Instance { mut limiter: Option<&mut StoreResourceLimiter<'_>>, module: &Module, imports: Imports<'_>, - asyncness: Asyncness, - ) -> Result<(Instance, Option)> { + ) -> Result { if !Engine::same(store.engine(), module.engine()) { bail!("cross-`Engine` instantiation is not currently supported"); } @@ -317,8 +320,6 @@ impl Instance { } } - let compiled_module = module.compiled_module(); - // Register the module just before instantiation to ensure we keep the module // properly referenced while in use by the store. let (modules, engine, breakpoints) = store.modules_and_engine_and_breakpoints_mut(); @@ -341,46 +342,12 @@ impl Instance { .await? }; - // Additionally, before we start doing fallible instantiation, we - // do one more step which is to insert an `InstanceData` - // corresponding to this instance. This `InstanceData` can be used - // via `Caller::get_export` if our instance's state "leaks" into - // other instances, even if we don't return successfully from this - // function. - // - // We don't actually load all exports from the instance at this - // time, instead preferring to lazily load them as they're demanded. - // For module/instance exports, though, those aren't actually - // stored in the instance handle so we need to immediately handle - // those here. - let instance = Instance::from_wasmtime(id, store); - - // Now that we've recorded all information we need to about this - // instance within a `Store` we can start performing fallible - // initialization. Note that we still defer the `start` function to - // later since that may need to run asynchronously. - // - // If this returns an error (or if the start function traps) then - // any other initialization which may have succeeded which placed - // items from this instance into other instances should be ok when - // those items are loaded and run we'll have all the metadata to - // look at them. - let bulk_memory = store - .engine() - .features() - .contains(WasmFeatures::BULK_MEMORY); - - vm::initialize_instance( - store, - limiter, - id, - compiled_module.module(), - bulk_memory, - asyncness, - ) - .await?; - - Ok((instance, compiled_module.module().start_func)) + // At this point the instance is created and stored within the store, + // but it's also not quite usable just yet. Initialization hasn't + // completed (e.g. active data/element segments) and the `start` + // function additionally has not yet been invoked. That's the + // responsibility of the caller to handle, however. + Ok(Instance::from_wasmtime(id, store)) } pub(crate) fn from_wasmtime(id: InstanceId, store: &mut StoreOpaque) -> Instance { @@ -389,7 +356,7 @@ impl Instance { } } - fn start_raw(&self, store: &mut StoreContextMut<'_, T>, start: FuncIndex) -> Result<()> { + fn start_raw(&self, store: &mut StoreContextMut<'_, T>) -> Result<()> { // If a start function is present, invoke it. Make sure we use all the // trap-handling configuration in `store` as well. let store_id = store.0.id(); @@ -399,7 +366,8 @@ impl Instance { let f = unsafe { instance .as_mut() - .get_exported_func(registry, store_id, start) + .get_startup_func(registry, store_id) + .expect("should have a startup function") }; let caller_vmctx = instance.vmctx(); unsafe { diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rs index 188d005e1fd7..8640cdf43acf 100644 --- a/crates/wasmtime/src/runtime/instantiate.rs +++ b/crates/wasmtime/src/runtime/instantiate.rs @@ -11,9 +11,8 @@ use alloc::sync::Arc; use core::ops::Range; use core::str; use wasmtime_environ::{ - CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, EntityRef, FilePos, FuncIndex, - FuncKey, FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex, - StaticModuleIndex, + CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, FilePos, FuncIndex, FuncKey, + FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex, StaticModuleIndex, }; /// A compiled wasm module, ready to be instantiated. @@ -116,55 +115,32 @@ impl CompiledModule { pub fn finished_function_ranges( &self, ) -> impl ExactSizeIterator)> + '_ { - self.module - .defined_func_indices() - .map(|i| (i, self.finished_function_range(i))) + self.module.defined_func_indices().map(|i| { + let key = FuncKey::DefinedWasmFunction(self.module_index(), i); + (i, self.function_range(key)) + }) } - /// Returns the offset in the text section of the function that `index` points to. + /// Returns the offset in the text section of the function that `key` + /// points to. #[inline] - pub fn finished_function_range(&self, def_func_index: DefinedFuncIndex) -> Range { - let loc = self.func_loc(def_func_index); + pub fn function_range(&self, key: FuncKey) -> Range { + let loc = self.func_loc(key); let start = usize::try_from(loc.start).unwrap(); let end = usize::try_from(loc.start + loc.length).unwrap(); start..end } - /// Get the array-to-Wasm trampoline for the function `index` - /// points to, as a range in the text segment. - /// - /// If the function `index` points to does not escape, then `None` is - /// returned. - /// - /// These trampolines are used for array callers (e.g. `Func::new`) - /// calling Wasm callees. - pub fn array_to_wasm_trampoline_range( - &self, - def_func_index: DefinedFuncIndex, - ) -> Option> { - assert!(def_func_index.index() < self.module.num_defined_funcs()); - let key = FuncKey::ArrayToWasmTrampoline(self.module_index(), def_func_index); - let loc = self.index.func_loc(key)?; - let start = usize::try_from(loc.start).unwrap(); - let end = usize::try_from(loc.start + loc.length).unwrap(); - Some(start..end) - } - /// Get the Wasm-to-array trampoline for the given signature, as a /// range in the text segment. /// /// These trampolines are used for filling in /// `VMFuncRef::wasm_call` for `Func::wrap`-style host funcrefs /// that don't have access to a compiler when created. - pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> Option<&[u8]> { + pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> &[u8] { let key = FuncKey::WasmToArrayTrampoline(signature); - let loc = self.index.func_loc(key)?; - let start = usize::try_from(loc.start).unwrap(); - let end = usize::try_from(loc.start + loc.length).unwrap(); - Some( - self.engine_code - .raw_wasm_to_array_trampoline_data(start..end), - ) + let range = self.function_range(key); + self.engine_code.raw_wasm_to_array_trampoline_data(range) } /// Lookups a defined function by a program counter value. @@ -190,9 +166,7 @@ impl CompiledModule { } /// Gets the function location information for a given function index. - pub fn func_loc(&self, def_func_index: DefinedFuncIndex) -> &FunctionLoc { - assert!(def_func_index.index() < self.module.num_defined_funcs()); - let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index); + pub fn func_loc(&self, key: FuncKey) -> &FunctionLoc { self.index .func_loc(key) .expect("defined function should be present") @@ -200,9 +174,7 @@ impl CompiledModule { /// Returns the original binary offset in the file that `index` was defined /// at. - pub fn func_start_srcloc(&self, def_func_index: DefinedFuncIndex) -> FilePos { - assert!(def_func_index.index() < self.module.num_defined_funcs()); - let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index); + pub fn func_start_srcloc(&self, key: FuncKey) -> FilePos { self.index .src_loc(key) .expect("defined function should be present") diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rs index 9b4294f34fcf..395225580417 100644 --- a/crates/wasmtime/src/runtime/module.rs +++ b/crates/wasmtime/src/runtime/module.rs @@ -19,8 +19,8 @@ use core::ptr::NonNull; use std::{fs::File, path::Path}; use wasmparser::{Parser, ValidPayload, Validator}; use wasmtime_environ::{ - CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind, - StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum, + CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, FuncKey, HostPtr, ModuleTypes, + ObjectKind, StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum, }; mod registry; @@ -1092,7 +1092,8 @@ impl Module { let module = self.compiled_module(); let module_index = self.env_module().module_index; self.env_module().defined_func_indices().map(move |idx| { - let loc = module.func_loc(idx); + let key = FuncKey::DefinedWasmFunction(module_index, idx); + let loc = module.func_loc(key); let idx = module.module().func_index(idx); ModuleFunction { module: module_index, @@ -1158,7 +1159,6 @@ impl Module { let ptr = self .compiled_module() .wasm_to_array_trampoline(trampoline_module_ty) - .expect("always have a trampoline for the trampoline type") .as_ptr() .cast::() .cast_mut(); @@ -1275,6 +1275,7 @@ mod tests { use wasmtime_environ::MemoryInitialization; #[test] + #[cfg_attr(miri, ignore)] fn cow_on_by_default() { let engine = Engine::default(); let module = Module::new( diff --git a/crates/wasmtime/src/runtime/profiling.rs b/crates/wasmtime/src/runtime/profiling.rs index 30234e32bcbe..336639c7983f 100644 --- a/crates/wasmtime/src/runtime/profiling.rs +++ b/crates/wasmtime/src/runtime/profiling.rs @@ -13,7 +13,7 @@ use fxprof_processed_profile::{ use std::ops::Range; use std::sync::Arc; use std::time::{Duration, Instant}; -use wasmtime_environ::demangle_function_name_or_index; +use wasmtime_environ::{FuncKey, demangle_function_name_or_index}; // TODO: collect more data // - On non-Windows, measure thread-local CPU usage between events with @@ -268,7 +268,9 @@ fn module_symbols(name: String, module: &Module) -> Option { .env_module() .defined_func_indices() .map(|defined_idx| { - let loc = compiled.func_loc(defined_idx); + let key = + FuncKey::DefinedWasmFunction(module.env_module().module_index, defined_idx); + let loc = compiled.func_loc(key); let func_idx = compiled.module().func_index(defined_idx); let mut name = String::new(); demangle_function_name_or_index( diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rs index cc49f4a7cf6f..e7d79cb4bf99 100644 --- a/crates/wasmtime/src/runtime/trap.rs +++ b/crates/wasmtime/src/runtime/trap.rs @@ -7,7 +7,7 @@ use core::fmt; use core::num::NonZeroUsize; use wasmtime_core::alloc::TryVec; use wasmtime_environ::{ - CompiledTrap, FilePos, demangle_function_name, demangle_function_name_or_index, + CompiledTrap, FilePos, FuncKey, demangle_function_name, demangle_function_name_or_index, }; /// Representation of a WebAssembly trap and what caused it to occur. @@ -501,7 +501,8 @@ impl FrameInfo { pub(crate) fn new(module: Module, text_offset: usize) -> Option { let compiled_module = module.compiled_module(); let index = compiled_module.func_by_text_offset(text_offset)?; - let func_start = compiled_module.func_start_srcloc(index); + let key = FuncKey::DefinedWasmFunction(compiled_module.module().module_index, index); + let func_start = compiled_module.func_start_srcloc(key); let instr = wasmtime_environ::lookup_file_pos(module.engine_code().address_map_data(), text_offset); let index = compiled_module.module().func_index(index); diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs index 832652a6949a..dd2830084df3 100644 --- a/crates/wasmtime/src/runtime/values.rs +++ b/crates/wasmtime/src/runtime/values.rs @@ -4,7 +4,6 @@ use crate::{ StructRef, V128, ValType, prelude::*, }; use core::ptr; -use wasmtime_environ::WasmHeapTopType; pub use crate::runtime::vm::ValRaw; @@ -125,16 +124,6 @@ impl Val { Val::AnyRef(None) } - pub(crate) const fn null_top(top: WasmHeapTopType) -> Val { - match top { - WasmHeapTopType::Func => Val::FuncRef(None), - WasmHeapTopType::Extern => Val::ExternRef(None), - WasmHeapTopType::Any => Val::AnyRef(None), - WasmHeapTopType::Exn => Val::ExnRef(None), - WasmHeapTopType::Cont => Val::ContRef(None), - } - } - /// Returns the default value for the given type, if any exists. /// /// Returns `None` if there is no default value for the given type (for diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 3a57c66547f3..a42ec6666bd6 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -53,7 +53,6 @@ use wasmtime_environ::ModuleInternedTypeIndex; mod always_mut; #[cfg(feature = "component-model")] pub mod component; -mod const_expr; mod export; mod gc; mod imports; @@ -100,7 +99,7 @@ pub use crate::runtime::vm::gc::*; pub use crate::runtime::vm::imports::Imports; pub use crate::runtime::vm::instance::{ GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, - MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, initialize_instance, + MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, }; #[cfg(feature = "pooling-allocator")] pub use crate::runtime::vm::instance::{ diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rs deleted file mode 100644 index a3d836bacbf9..000000000000 --- a/crates/wasmtime/src/runtime/vm/const_expr.rs +++ /dev/null @@ -1,499 +0,0 @@ -//! Evaluating const expressions. - -use crate::prelude::*; -use crate::runtime::vm; -use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter}; -#[cfg(feature = "gc")] -use crate::{ - AnyRef, ArrayRef, ArrayRefPre, ArrayType, ExternRef, I31, StructRef, StructRefPre, StructType, -}; -use crate::{OpaqueRootScope, Val}; -use wasmtime_environ::{ConstExpr, ConstOp, FuncIndex, GlobalConstValue, GlobalIndex}; -#[cfg(feature = "gc")] -use wasmtime_environ::{VMSharedTypeIndex, WasmCompositeInnerType, WasmCompositeType, WasmSubType}; - -/// An interpreter for const expressions. -/// -/// This can be reused across many const expression evaluations to reuse -/// allocated resources, if any. -pub struct ConstExprEvaluator { - stack: TryVec, - simple: Val, -} - -impl Default for ConstExprEvaluator { - fn default() -> ConstExprEvaluator { - ConstExprEvaluator { - stack: TryVec::new(), - simple: Val::I32(0), - } - } -} - -/// The context within which a particular const expression is evaluated. -pub struct ConstEvalContext { - pub(crate) instance: InstanceId, - #[cfg_attr( - not(feature = "gc"), - expect(dead_code, reason = "easier than conditionally compiling this field") - )] - pub(crate) asyncness: Asyncness, -} - -impl ConstEvalContext { - /// Create a new context. - pub fn new(instance: InstanceId, asyncness: Asyncness) -> Self { - Self { - instance, - asyncness, - } - } - - fn global_get(&mut self, store: &mut StoreOpaque, index: GlobalIndex) -> Result { - let id = store.id(); - Ok(store - .instance_mut(self.instance) - .get_exported_global(id, index) - ._get(&mut AutoAssertNoGc::new(store))) - } - - fn ref_func(&mut self, store: &mut StoreOpaque, index: FuncIndex) -> Result { - let id = store.id(); - let (instance, registry) = store.instance_and_module_registry_mut(self.instance); - // SAFETY: `id` is the correct store-owner of the function being looked - // up - let func = unsafe { instance.get_exported_func(registry, id, index) }; - Ok(func.into()) - } - - #[cfg(feature = "gc")] - fn struct_fields_len(&self, store: &mut StoreOpaque, shared_ty: VMSharedTypeIndex) -> usize { - let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty); - let fields = struct_ty.fields(); - fields.len() - } - - #[cfg(feature = "gc")] - async fn struct_new( - &mut self, - store: &mut StoreOpaque, - limiter: Option<&mut StoreResourceLimiter<'_>>, - shared_ty: VMSharedTypeIndex, - fields: &[Val], - ) -> Result { - let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty); - let allocator = StructRefPre::_new(store, struct_ty); - let struct_ref = - StructRef::_new_async(store, limiter, &allocator, &fields, self.asyncness).await?; - Ok(Val::AnyRef(Some(struct_ref.into()))) - } - - #[cfg(feature = "gc")] - async fn struct_new_default( - &mut self, - store: &mut StoreOpaque, - limiter: Option<&mut StoreResourceLimiter<'_>>, - shared_ty: VMSharedTypeIndex, - ) -> Result { - let module = store - .instance(self.instance) - .runtime_module() - .expect("should never be allocating a struct type defined in a dummy module"); - - let borrowed = module - .engine() - .signatures() - .borrow(shared_ty) - .expect("should have a registered type for struct"); - let WasmSubType { - composite_type: - WasmCompositeType { - shared: false, - inner: WasmCompositeInnerType::Struct(struct_ty), - }, - .. - } = &*borrowed - else { - unreachable!("registered type should be a struct"); - }; - - let fields = struct_ty - .fields - .iter() - .map(|ty| match &ty.element_type { - wasmtime_environ::WasmStorageType::I8 | wasmtime_environ::WasmStorageType::I16 => { - Val::I32(0) - } - wasmtime_environ::WasmStorageType::Val(v) => match v { - wasmtime_environ::WasmValType::I32 => Val::I32(0), - wasmtime_environ::WasmValType::I64 => Val::I64(0), - wasmtime_environ::WasmValType::F32 => Val::F32(0.0f32.to_bits()), - wasmtime_environ::WasmValType::F64 => Val::F64(0.0f64.to_bits()), - wasmtime_environ::WasmValType::V128 => Val::V128(0u128.into()), - wasmtime_environ::WasmValType::Ref(r) => { - assert!(r.nullable); - Val::null_top(r.heap_type.top()) - } - }, - }) - .collect::>(); - - self.struct_new(store, limiter, shared_ty, &fields).await - } -} - -impl ConstExprEvaluator { - /// Same as [`Self::eval`] except that this is specifically intended for - /// integral constant expression. - /// - /// # Panics - /// - /// Panics if `ConstExpr` contains GC ops (e.g. it's not for an integral - /// type). - pub fn eval_int( - &mut self, - store: &mut StoreOpaque, - context: &mut ConstEvalContext, - expr: &ConstExpr, - ) -> Result<&Val> { - // Try to evaluate a simple expression first before doing the more - // complicated eval loop below. - if self.try_simple(expr).is_some() { - return Ok(&self.simple); - } - - // Note that `assert_ready` here should be valid as production of an - // integer cannot involve GC meaning that async operations aren't used. - let mut scope = OpaqueRootScope::new(store); - vm::assert_ready(self.eval_loop(&mut scope, None, context, expr)) - } - - /// Attempts to peek into `expr` to see if it's trivial to evaluate, e.g. - /// for `i32.const N`. - #[inline] - pub fn try_simple(&mut self, expr: &ConstExpr) -> Option<&Val> { - match expr.const_eval()? { - GlobalConstValue::I32(i) => Some(self.return_one(Val::I32(i))), - GlobalConstValue::I64(i) => Some(self.return_one(Val::I64(i))), - GlobalConstValue::F32(f) => Some(self.return_one(Val::F32(f))), - GlobalConstValue::F64(f) => Some(self.return_one(Val::F64(f))), - GlobalConstValue::V128(i) => Some(self.return_one(Val::V128(i.into()))), - } - } - - /// Evaluate the given const expression in the given context. - /// - /// Note that the `store` argument is an `OpaqueRootScope` which is used to - /// require that a GC rooting scope external to evaluation of this constant - /// is required. Constant expression evaluation may perform GC allocations - /// and itself trigger a GC meaning that all references must be rooted, - /// hence the external requirement of a rooting scope. - /// - /// # Panics - /// - /// This function will panic if `expr` is an invalid constant expression. - pub async fn eval( - &mut self, - store: &mut OpaqueRootScope<&mut StoreOpaque>, - limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - expr: &ConstExpr, - ) -> Result<&Val> { - // Same structure as `eval_int` above, except using `.await` and with a - // slightly different type signature here for this function. - if self.try_simple(expr).is_some() { - return Ok(&self.simple); - } - self.eval_loop(store, limiter, context, expr).await - } - - #[inline] - fn return_one(&mut self, val: Val) -> &Val { - self.simple = val; - &self.simple - } - - #[cold] - async fn eval_loop( - &mut self, - store: &mut OpaqueRootScope<&mut StoreOpaque>, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - expr: &ConstExpr, - ) -> Result<&Val> { - log::trace!("evaluating const expr: {expr:?}"); - - self.stack.clear(); - - // On GC-less builds ensure that this is always considered used an - // needed-mutable. - let _ = &mut limiter; - - for op in expr.ops() { - log::trace!("const-evaluating op: {op:?}"); - match op { - ConstOp::I32Const(i) => self.stack.push(Val::I32(*i))?, - ConstOp::I64Const(i) => self.stack.push(Val::I64(*i))?, - ConstOp::F32Const(f) => self.stack.push(Val::F32(*f))?, - ConstOp::F64Const(f) => self.stack.push(Val::F64(*f))?, - ConstOp::V128Const(v) => self.stack.push(Val::V128((*v).into()))?, - ConstOp::GlobalGet(g) => self.stack.push(context.global_get(store, *g)?)?, - ConstOp::RefNull(ty) => self.stack.push(Val::null_top(*ty))?, - ConstOp::RefFunc(f) => self.stack.push(context.ref_func(store, *f)?)?, - #[cfg(feature = "gc")] - ConstOp::RefI31 => { - let i = self.pop()?.unwrap_i32(); - let i31 = I31::wrapping_i32(i); - let r = AnyRef::_from_i31(&mut AutoAssertNoGc::new(store), i31); - self.stack.push(Val::AnyRef(Some(r)))?; - } - #[cfg(not(feature = "gc"))] - ConstOp::RefI31 => panic!("should not have validated"), - ConstOp::I32Add => { - let b = self.pop()?.unwrap_i32(); - let a = self.pop()?.unwrap_i32(); - self.stack.push(Val::I32(a.wrapping_add(b)))?; - } - ConstOp::I32Sub => { - let b = self.pop()?.unwrap_i32(); - let a = self.pop()?.unwrap_i32(); - self.stack.push(Val::I32(a.wrapping_sub(b)))?; - } - ConstOp::I32Mul => { - let b = self.pop()?.unwrap_i32(); - let a = self.pop()?.unwrap_i32(); - self.stack.push(Val::I32(a.wrapping_mul(b)))?; - } - ConstOp::I64Add => { - let b = self.pop()?.unwrap_i64(); - let a = self.pop()?.unwrap_i64(); - self.stack.push(Val::I64(a.wrapping_add(b)))?; - } - ConstOp::I64Sub => { - let b = self.pop()?.unwrap_i64(); - let a = self.pop()?.unwrap_i64(); - self.stack.push(Val::I64(a.wrapping_sub(b)))?; - } - ConstOp::I64Mul => { - let b = self.pop()?.unwrap_i64(); - let a = self.pop()?.unwrap_i64(); - self.stack.push(Val::I64(a.wrapping_mul(b)))?; - } - - #[cfg(not(feature = "gc"))] - ConstOp::StructNew { .. } - | ConstOp::StructNewDefault { .. } - | ConstOp::ArrayNew { .. } - | ConstOp::ArrayNewDefault { .. } - | ConstOp::ArrayNewFixed { .. } - | ConstOp::ExternConvertAny - | ConstOp::AnyConvertExtern => { - bail!( - "const expr evaluation error: struct operations are not \ - supported without the `gc` feature" - ) - } - - #[cfg(feature = "gc")] - ConstOp::StructNew { struct_type_index } => { - let interned_type_index = store.instance(context.instance).env_module().types - [*struct_type_index] - .unwrap_engine_type_index(); - let len = context.struct_fields_len(store, interned_type_index); - - if self.stack.len() < len { - bail!( - "const expr evaluation error: expected at least {len} values on the stack, found {}", - self.stack.len() - ) - } - - let start = self.stack.len() - len; - let s = context - .struct_new( - store, - limiter.as_deref_mut(), - interned_type_index, - &self.stack[start..], - ) - .await?; - self.stack.truncate(start); - self.stack.push(s)?; - } - - #[cfg(feature = "gc")] - ConstOp::StructNewDefault { struct_type_index } => { - let ty = store.instance(context.instance).env_module().types - [*struct_type_index] - .unwrap_engine_type_index(); - self.stack.push( - context - .struct_new_default(store, limiter.as_deref_mut(), ty) - .await?, - )?; - } - - #[cfg(feature = "gc")] - ConstOp::ArrayNew { array_type_index } => { - let ty = store.instance(context.instance).env_module().types[*array_type_index] - .unwrap_engine_type_index(); - let ty = ArrayType::from_shared_type_index(store.engine(), ty); - - let len = self.pop()?.unwrap_i32().cast_unsigned(); - - let elem = self.pop()?; - - let pre = ArrayRefPre::_new(store, ty); - let array = ArrayRef::_new_async( - store, - limiter.as_deref_mut(), - &pre, - &elem, - len, - context.asyncness, - ) - .await?; - - self.stack.push(Val::AnyRef(Some(array.into())))?; - } - - #[cfg(feature = "gc")] - ConstOp::ArrayNewDefault { array_type_index } => { - let ty = store.instance(context.instance).env_module().types[*array_type_index] - .unwrap_engine_type_index(); - let ty = ArrayType::from_shared_type_index(store.engine(), ty); - - let len = self.pop()?.unwrap_i32().cast_unsigned(); - - let elem = Val::default_for_ty(ty.element_type().unpack()) - .expect("type should have a default value"); - - let pre = ArrayRefPre::_new(store, ty); - let array = ArrayRef::_new_async( - store, - limiter.as_deref_mut(), - &pre, - &elem, - len, - context.asyncness, - ) - .await?; - - self.stack.push(Val::AnyRef(Some(array.into())))?; - } - - #[cfg(feature = "gc")] - ConstOp::ArrayNewFixed { - array_type_index, - array_size, - } => { - let ty = store.instance(context.instance).env_module().types[*array_type_index] - .unwrap_engine_type_index(); - let ty = ArrayType::from_shared_type_index(store.engine(), ty); - - let array_size = usize::try_from(*array_size).unwrap(); - if self.stack.len() < array_size { - bail!( - "const expr evaluation error: expected at least {array_size} values on the stack, found {}", - self.stack.len() - ) - } - - let start = self.stack.len() - array_size; - - let elems = self - .stack - .drain(start..) - .collect::>(); - - let pre = ArrayRefPre::_new(store, ty); - let array = ArrayRef::_new_fixed_async( - store, - limiter.as_deref_mut(), - &pre, - &elems, - context.asyncness, - ) - .await?; - - self.stack.push(Val::AnyRef(Some(array.into())))?; - } - - #[cfg(feature = "gc")] - ConstOp::ExternConvertAny => { - let mut store = AutoAssertNoGc::new(store); - let result = match self.pop()?.unwrap_anyref() { - Some(anyref) => Some(ExternRef::_convert_any(&mut store, *anyref)?), - None => None, - }; - self.stack.push(Val::ExternRef(result))?; - } - - #[cfg(feature = "gc")] - ConstOp::AnyConvertExtern => { - let mut store = AutoAssertNoGc::new(store); - let result = match self.pop()?.unwrap_externref() { - Some(externref) => Some(AnyRef::_convert_extern(&mut store, *externref)?), - None => None, - }; - self.stack.push(result.into())?; - } - } - } - - if self.stack.len() == 1 { - log::trace!("const expr evaluated to {:?}", self.stack[0]); - Ok(&self.stack[0]) - } else { - bail!( - "const expr evaluation error: expected 1 resulting value, found {}", - self.stack.len() - ) - } - } - - fn pop(&mut self) -> Result { - self.stack.pop().ok_or_else(|| { - format_err!( - "const expr evaluation error: attempted to pop from an empty \ - evaluation stack" - ) - }) - } -} - -#[cfg(test)] -mod tests { - use super::{ConstExprEvaluator, Val}; - use wasmtime_environ::{ConstExpr, ConstOp}; - - #[test] - fn pop_empty_stack() { - let mut evaluator = ConstExprEvaluator::default(); - let err = evaluator.pop().unwrap_err(); - assert!( - format!("{err}").contains("attempted to pop from an empty"), - "unexpected error: {err}" - ); - } - - #[test] - fn try_simple_scalar_types() { - let mut ev = ConstExprEvaluator::default(); - - let expr = ConstExpr::new([ConstOp::I32Const(7)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::I32(7)))); - - let expr = ConstExpr::new([ConstOp::I64Const(99)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::I64(99)))); - - let expr = ConstExpr::new([ConstOp::F32Const(0x3f800000)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::F32(_)))); - - let expr = ConstExpr::new([ConstOp::F64Const(0x3ff0000000000000)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::F64(_)))); - - let expr = ConstExpr::new([ConstOp::V128Const(0x12345678)]); - assert!(matches!(ev.try_simple(&expr), Some(Val::V128(_)))); - } -} diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rs index 15dbbaae5db4..6f8129f47c6a 100644 --- a/crates/wasmtime/src/runtime/vm/cow.rs +++ b/crates/wasmtime/src/runtime/vm/cow.rs @@ -194,7 +194,7 @@ impl ModuleMemoryImages { // If there's no initialization for this memory known then we don't // need an image for the memory so push `None` and move on. - let init = match init { + let (offset, runtime_index) = match init { Some(init) => init, None => { memories.push(None)?; @@ -202,11 +202,14 @@ impl ModuleMemoryImages { } }; - let data_range = init.data.start as usize..init.data.end as usize; + let data_range = &module.runtime_data[*runtime_index]; + let data_range = usize::try_from(data_range.start).unwrap() + ..usize::try_from(data_range.end).unwrap(); + if module.memories[memory_index] .minimum_byte_size() .map_or(false, |mem_initial_len| { - init.offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len + *offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len }) { // The image is rounded up to multiples of the host OS page @@ -220,7 +223,7 @@ impl ModuleMemoryImages { return Ok(None); } - let offset_usize = match usize::try_from(init.offset) { + let offset_usize = match usize::try_from(*offset) { Ok(offset) => offset, Err(_) => return Ok(None), }; diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs index 38057577f108..61be8250cda1 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs @@ -1370,7 +1370,8 @@ mod tests { num_defined_globals: 0, num_defined_tags: 0, num_escaped_funcs: 0, - num_passive_data: 0, + num_runtime_data: 0, + has_startup_func: false, }); assert_eq!( diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs index 9b097c43e7d5..600d3c1a8820 100644 --- a/crates/wasmtime/src/runtime/vm/instance.rs +++ b/crates/wasmtime/src/runtime/vm/instance.rs @@ -2,7 +2,6 @@ //! wasm module (except its callstack and register state). An //! `InstanceHandle` is a reference-counting handle for an `Instance`. -use crate::Val; use crate::code::ModuleWithCode; use crate::module::ModuleRegistry; use crate::prelude::*; @@ -18,9 +17,7 @@ use crate::runtime::vm::{ GcStore, HostResult, Imports, ModuleRuntimeInfo, SendSyncPtr, VMGcRef, VMGlobalKind, VMStore, VMStoreRawPtr, VmPtr, VmSafe, WasmFault, catch_unwind_and_record_trap, }; -use crate::store::{ - AutoAssertNoGc, InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter, -}; +use crate::store::{InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter}; use crate::vm::{VMWasmCallFunction, ValRaw}; use alloc::sync::Arc; use core::alloc::Layout; @@ -35,10 +32,11 @@ use core::{mem, ptr}; use wasmtime_environ::ModuleInternedTypeIndex; use wasmtime_environ::error::OutOfMemory; use wasmtime_environ::{ - DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, EntityIndex, - EntityRef, FuncIndex, GlobalIndex, HostPtr, MemoryIndex, PassiveDataIndex, PassiveElemIndex, - PtrSize, TableIndex, TableInitialValue, TagIndex, VMCONTEXT_MAGIC, VMOffsets, - VMSharedTypeIndex, WasmRefType, packed_option::ReservedValue, + Abi, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, + DefinedTagIndex, EntityIndex, EntityRef, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex, + HostPtr, MemoryIndex, MemoryInitialization, ModuleStartup, PassiveElemIndex, PtrSize, + RuntimeDataIndex, TableIndex, TagIndex, VMCONTEXT_MAGIC, VMOffsets, VMSharedTypeIndex, + WasmRefType, packed_option::ReservedValue, }; #[cfg(feature = "wmemcheck")] use wasmtime_wmemcheck::Wmemcheck; @@ -170,7 +168,7 @@ impl Instance { ) -> Result { let module = req.runtime_info.env_module(); let memory_tys = &module.memories; - let passive_elements = TryVec::with_capacity(module.passive_elements.len())?; + let mut passive_elements = TryVec::with_capacity(module.passive_elements.len())?; #[cfg(feature = "wmemcheck")] let wmemcheck_state = if req.store.engine().config().wmemcheck { @@ -188,6 +186,11 @@ impl Instance { #[cfg(not(feature = "wmemcheck"))] let _ = memory_tys; + for (_, (ty, len)) in req.runtime_info.env_module().passive_elements.iter() { + let len = usize::try_from(*len).unwrap(); + passive_elements.push(PassiveElementSegment::new(*ty, len)?)?; + } + let mut ret = OwnedInstance::new(Instance { id: req.id, runtime_info: req.runtime_info.clone(), @@ -205,6 +208,7 @@ impl Instance { unsafe { ret.get_mut().initialize_vmctx(req.store, req.imports); } + Ok(ret) } @@ -447,6 +451,7 @@ impl Instance { } /// Get a locally defined or imported memory. + #[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))] pub(crate) fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition { if let Some(defined_index) = self.env_module().defined_memory_index(index) { self.memory(defined_index) @@ -594,6 +599,26 @@ impl Instance { unsafe { crate::Func::from_vm_func_ref(store, func_ref) } } + /// Returns a `Func` corresponding to the startup function for this + /// instance, if generated at compile time. + /// + /// # Safety + /// + /// The `store` parameter must be the store that owns this instance and the + /// functions that this instance can reference. + pub unsafe fn get_startup_func( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + store: StoreId, + ) -> Option { + let func_ref = self.get_start_func_ref(registry)?; + + // SAFETY: the validity of `func_ref` is guaranteed by the validity of + // `self`, and the contract that `store` must own `func_ref` is a + // contract of this function itself. + Some(unsafe { crate::Func::from_vm_func_ref(store, func_ref) }) + } + /// Lookup a table by index. /// /// # Panics @@ -820,11 +845,86 @@ impl Instance { return None; } - let Some(def_index) = self.env_module().defined_func_index(index) else { - debug_assert!(self.env_module().is_imported_function(index)); - return Some(self.imported_function(index).as_func_ref().into()); + match self.env_module().defined_func_index(index) { + Some(index) => self.initialize_defined_funcref(registry, index), + None => { + debug_assert!(self.env_module().is_imported_function(index)); + Some(self.imported_function(index).as_func_ref().into()) + } + } + } + + /// Initializes a defined function's `VMFuncRef` in-place and then returns a + /// pointer to that location. + fn initialize_defined_funcref( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + def_index: DefinedFuncIndex, + ) -> Option> { + let module = self.env_module(); + let index = module.func_index(def_index); + let func = &module.functions[index]; + let type_index = func.signature.unwrap_engine_type_index(); + let vmctx_offset = self.offsets().vmctx_func_ref(func.func_ref); + let array_to_wasm_key = FuncKey::ArrayToWasmTrampoline(module.module_index, def_index); + let wasm_key = FuncKey::DefinedWasmFunction(module.module_index, def_index); + // SAFETY: the type/offset/keys here are all valid for the defined + // function at `def_index`. + unsafe { + self.initialize_and_return_funcref( + registry, + type_index, + vmctx_offset, + array_to_wasm_key, + wasm_key, + ) + } + } + + fn get_start_func_ref( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + ) -> Option> { + let module = self.env_module(); + let type_index = match module.startup { + ModuleStartup::None => return None, + ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => { + t.unwrap_engine_type_index() + } }; + let vmctx_offset = self.offsets().vmctx_startup_func_ref(); + let array_to_wasm_key = FuncKey::ModuleStartup(Abi::Array, module.module_index); + let wasm_key = FuncKey::ModuleStartup(Abi::Wasm, module.module_index); + // SAFETY: the type/offset/keys here are all valid for the module + // startup function. + unsafe { + self.initialize_and_return_funcref( + registry, + type_index, + vmctx_offset, + array_to_wasm_key, + wasm_key, + ) + } + } + /// Common implementation of initializing a `VMFuncRef` stored within this + /// instance's `VMContext`. + /// + /// # Safety + /// + /// This function requires that `type_index` accurately describes this + /// function and `vmctx_offset` is indeed the correct offset for the + /// functions here. Effectively all the arguments here must be "logically + /// correct" for the `VMFuncRef` being initialized. + unsafe fn initialize_and_return_funcref( + self: Pin<&mut Self>, + registry: &ModuleRegistry, + type_index: VMSharedTypeIndex, + vmctx_offset: u32, + array_to_wasm_key: FuncKey, + wasm_key: FuncKey, + ) -> Option> { // For now, we eagerly initialize an funcref struct in-place whenever // asked for a reference to it. This is mostly fine, because in practice // each funcref is unlikely to be requested more than a few times: @@ -846,9 +946,6 @@ impl Instance { // it's better for instantiation performance if we don't have to track // "is-initialized" state at all! - let func = &self.env_module().functions[index]; - let type_index = func.signature.unwrap_engine_type_index(); - let module_with_code = ModuleWithCode::in_store( registry, self.runtime_module() @@ -856,19 +953,13 @@ impl Instance { ) .expect("module not in store"); - let array_call = VmPtr::from( - NonNull::from( - module_with_code - .array_to_wasm_trampoline(def_index) - .expect("should have array-to-Wasm trampoline for escaping function"), - ) - .cast(), - ); + let array_call = + VmPtr::from(NonNull::from(module_with_code.function(array_to_wasm_key)).cast()); let wasm_call = Some(VmPtr::from( NonNull::new( module_with_code - .finished_function(def_index) + .function(wasm_key) .as_ptr() .cast::() .cast_mut(), @@ -880,9 +971,7 @@ impl Instance { // SAFETY: the offset calculated here should be correct with // `self.offsets` - let func_ref_ptr = unsafe { - self.vmctx_plus_offset_raw::(self.offsets().vmctx_func_ref(func.func_ref)) - }; + let func_ref_ptr = unsafe { self.vmctx_plus_offset_raw::(vmctx_offset) }; // SAFETY: the `func_ref_ptr` should be valid as it's within our // `VMContext` area. @@ -899,15 +988,16 @@ impl Instance { } /// Get the passive elements segment at the given index. - pub(crate) fn passive_element_segment(&self, passive: PassiveElemIndex) -> &[ValRaw] { - self.passive_elements[passive.index()].elements() + pub(crate) fn passive_element_segment( + self: Pin<&mut Self>, + passive: PassiveElemIndex, + ) -> &mut [ValRaw] { + self.passive_elements_mut()[passive.index()].elements_mut() } - pub(crate) fn passive_elements_mut( - self: Pin<&mut Self>, - ) -> Pin<&mut TryVec> { + pub(crate) fn passive_elements_mut(self: Pin<&mut Self>) -> &mut TryVec { // SAFETY: Not moving data out of `self`. - Pin::new(&mut unsafe { self.get_unchecked_mut() }.passive_elements) + &mut unsafe { self.get_unchecked_mut() }.passive_elements } /// Drop an element. @@ -946,7 +1036,7 @@ impl Instance { &self.runtime_info.wasm_data()[start..end] } - /// Returns the data for the passive segment identified by `index` + /// Returns the data for the runtime segment identified by `index` /// /// Does not take into account the dynamic size of the data pointed to by /// `index`, always returns the raw data from the module itself. @@ -954,8 +1044,8 @@ impl Instance { /// # Panics /// /// Panics if `index` is out-of-bounds. - fn passive_data(&self, index: PassiveDataIndex) -> &[u8] { - let range = self.env_module().passive_data[index].clone(); + fn runtime_data(&self, index: RuntimeDataIndex) -> &[u8] { + let range = self.env_module().runtime_data[index].clone(); self.wasm_data(range) } @@ -1013,10 +1103,7 @@ impl Instance { // that `i` may be outside the limits of the static // initialization so it's a fallible `get` instead of an index. let module = self.env_module(); - let precomputed = match &module.table_initialization.initial_values[idx] { - TableInitialValue::Null { precomputed } => precomputed, - TableInitialValue::Expr(_) => unreachable!(), - }; + let precomputed = &module.table_initialization[idx]; // Panicking here helps catch bugs rather than silently truncating by accident. let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned(); let func_ref = func_index @@ -1031,13 +1118,6 @@ impl Instance { self.get_defined_table(idx) } - /// Get a table by index regardless of whether it is locally-defined or an - /// imported, foreign table. - pub(crate) fn get_table(self: Pin<&mut Self>, table_index: TableIndex) -> &mut Table { - let (idx, instance) = self.defined_table_index_and_instance(table_index); - instance.get_defined_table(idx) - } - /// Get a locally-defined table. pub(crate) fn get_defined_table(self: Pin<&mut Self>, index: DefinedTableIndex) -> &mut Table { &mut self.tables_mut()[index].1 @@ -1237,9 +1317,21 @@ impl Instance { // after this, but if it's read then it'd hopefully crash faster than // leaving this undefined. unsafe { - for (index, _init) in module.global_initializers.iter() { + for i in 0..module.num_defined_globals() { + let index = DefinedGlobalIndex::new(i); instance.global_ptr(index).write(VMGlobalDefinition::new()); } + for (index, val) in module.global_initializers.iter() { + let mut def = VMGlobalDefinition::new(); + match val { + GlobalConstValue::I32(i) => *def.as_i32_mut() = *i, + GlobalConstValue::I64(i) => *def.as_i64_mut() = *i, + GlobalConstValue::F32(i) => *def.as_f32_bits_mut() = *i, + GlobalConstValue::F64(i) => *def.as_f64_bits_mut() = *i, + GlobalConstValue::V128(i) => def.set_u128(*i), + } + instance.global_ptr(*index).write(def); + } } // Initialize the defined tags @@ -1261,7 +1353,7 @@ impl Instance { } } - // Initialize the lengths of passive data segments. + // Initialize the lengths of runtime data segments. // // SAFETY: it's safe to initialize these lengths during initialization // here and the various types of pointers and such here should all be @@ -1269,17 +1361,48 @@ impl Instance { unsafe { let offsets = instance.runtime_info.offsets(); let mut lengths = - instance.vmctx_plus_offset_raw(offsets.vmctx_passive_data_lengths_begin()); + instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_lengths_begin()); let mut bases = - instance.vmctx_plus_offset_raw(offsets.vmctx_passive_data_bases_begin()); - for i in module.passive_data.keys() { - let data = instance.passive_data(i); + instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_bases_begin()); + for i in module.runtime_data.keys() { + let data = instance.runtime_data(i); lengths.write(u32::try_from(data.len()).unwrap()); lengths = lengths.add(1); bases.write(VmPtr::from(NonNull::from(data).cast::())); bases = bases.add(1); } } + + // This is the half of the strategy of implementing memory-init-cow data + // segments. Notably the compiled startup function, if present, will + // skip data segments that have a null pointer. Here each linear memory + // is tested to see if it needs initialization. If it does, then the + // data segment is left in-place (and the startup function will + // initialize linear memory). Otherwise the data segment is null'd out. + // If the startup function runs (e.g. something else in the module needs + // it), then the corresponding data segment's initialization will be + // skipped. + if let MemoryInitialization::Static { map } = &module.memory_initialization { + for (memory, init) in map { + let Some(memory) = module.defined_memory_index(memory) else { + continue; + }; + if instance.memories[memory].1.needs_init() { + continue; + } + if let Some((_offset, data)) = init { + let offsets = instance.runtime_info.offsets(); + unsafe { + instance + .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_length(*data)) + .write(0u32); + instance + .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_base(*data)) + .write(0usize); + } + } + } + } } /// Attempts to convert from the host `addr` specified to a WebAssembly @@ -1397,6 +1520,17 @@ impl Instance { // SAFETY: see `store_mut` above. unsafe { &mut self.get_unchecked_mut().wmemcheck_state } } + + pub(crate) fn needs_startup(&self) -> bool { + match self.env_module().startup { + ModuleStartup::None => false, + ModuleStartup::Always(_) => true, + ModuleStartup::IfMemoriesNeedInit(_) => self + .memories + .iter() + .any(|(_, (_, memory))| memory.needs_init()), + } + } } // SAFETY: `layout` should describe this accurately and `OwnedVMContext` is the @@ -1721,39 +1855,14 @@ pub(crate) struct PassiveElementSegment { impl PassiveElementSegment { /// Create a new passive element segment with the given capacity. pub(crate) fn new(ty: WasmRefType, capacity: usize) -> Result { + let mut elements = TryVec::with_capacity(capacity)?; + elements.resize_with(capacity, || ValRaw::null())?; Ok(Self { needs_gc_rooting: ty.is_vmgcref_type_and_not_i31(), - elements: TryVec::with_capacity(capacity)?, + elements, }) } - /// Push a value onto this passive element segment. - /// - /// NB: Does not type check the value, relies on callers to ensure the - /// value is of the correct type (generally, due to validation). - pub(crate) fn push(&mut self, store: &mut StoreOpaque, val: Val) -> Result<()> { - let mut val = { - let mut store = AutoAssertNoGc::new(store); - val.to_raw_(&mut store)? - }; - if self.needs_gc_rooting { - // Note that `anyref` accessors and constructors are used here - // without actually checking the type of this segment or value. The - // representation and handling of all three is the same which means - // that this should work out. - let gc_ref = val.get_anyref(); - debug_assert_eq!(gc_ref, val.get_exnref()); - debug_assert_eq!(gc_ref, val.get_externref()); - if let Some(gc_ref) = VMGcRef::from_raw_u32(gc_ref) { - if let Some(gc_store) = store.optional_gc_store_mut() { - val = ValRaw::anyref(gc_store.clone_gc_ref(&gc_ref).as_raw_u32()); - } - } - } - self.elements.push(val)?; - Ok(()) - } - /// Clear this segment's elements. pub(crate) fn clear(&mut self, mut gc_store: Option<&mut GcStore>) { let elements = mem::take(&mut self.elements); @@ -1776,12 +1885,6 @@ impl PassiveElementSegment { } /// The elements of this segment. - pub(crate) fn elements(&self) -> &[ValRaw] { - &self.elements - } - - /// The elements of this segment. - #[cfg(feature = "gc")] pub(crate) fn elements_mut(&mut self) -> &mut [ValRaw] { &mut self.elements } diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rs index e2f6ae082455..47e301ede32f 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs @@ -1,21 +1,16 @@ use crate::prelude::*; -use crate::runtime::vm::const_expr::{ConstEvalContext, ConstExprEvaluator}; use crate::runtime::vm::imports::Imports; use crate::runtime::vm::instance::{Instance, InstanceHandle}; use crate::runtime::vm::memory::Memory; use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::table::Table; use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo}; -use crate::store::{Asyncness, InstanceId, StoreOpaque, StoreResourceLimiter}; -use crate::vm::instance::PassiveElementSegment; -use crate::{OpaqueRootScope, Val}; +use crate::store::{InstanceId, StoreOpaque, StoreResourceLimiter}; use core::future::Future; +use core::mem; use core::pin::Pin; -use core::{mem, ptr}; use wasmtime_environ::{ - DefinedMemoryIndex, DefinedTableIndex, EntityRef, HostPtr, InitMemory, MemoryInitialization, - MemoryInitializer, MemoryKind, Module, SizeOverflow, TableInitialValue, TableSegmentElements, - Trap, VMOffsets, WasmRefType, + DefinedMemoryIndex, DefinedTableIndex, HostPtr, MemoryKind, Module, VMOffsets, }; #[cfg(feature = "gc")] @@ -499,445 +494,6 @@ impl dyn InstanceAllocator + '_ { } } -fn check_table_init_bounds( - store: &mut StoreOpaque, - instance: InstanceId, - module: &Module, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, -) -> Result<()> { - let mut store = OpaqueRootScope::new(store); - - for segment in module.table_initialization.segments.iter() { - let start = const_evaluator.eval_int(&mut store, context, &segment.offset)?; - let start = get_index(start, module.tables[segment.table_index].idx_type); - let end = start.checked_add(segment.elements.len()); - - let table = store.instance_mut(instance).get_table(segment.table_index); - match end { - Some(end) if end <= u64::try_from(table.size())? => { - // Initializer is in bounds - } - _ => { - bail!(Trap::TableOutOfBounds); - } - } - } - - Ok(()) -} - -async fn initialize_tables( - store: &mut StoreOpaque, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - let mut store = OpaqueRootScope::new(store); - for (table, init) in module.table_initialization.initial_values.iter() { - match init { - // Tables are always initially null-initialized at this time - TableInitialValue::Null { precomputed: _ } => {} - - TableInitialValue::Expr(expr) => { - let init = const_evaluator - .eval(&mut store, limiter.as_deref_mut(), context, expr) - .await?; - let idx = module.table_index(table); - let id = store.id(); - let table = store - .instance_mut(context.instance) - .get_exported_table(id, idx); - let size = table.size_(&store); - table._fill(&mut store, 0, init.ref_().unwrap(), size)?; - } - } - } - - // Note: if the module's table initializer state is in - // FuncTable mode, we will lazily initialize tables based on - // any statically-precomputed image of FuncIndexes, but there - // may still be "leftover segments" that could not be - // incorporated. So we have a unified handler here that - // iterates over all segments (Segments mode) or leftover - // segments (FuncTable mode) to initialize. - for segment in module.table_initialization.segments.iter() { - let start = const_evaluator.eval_int(&mut store, context, &segment.offset)?; - let start = get_index(start, module.tables[segment.table_index].idx_type); - - let end = start - .checked_add(segment.elements.len()) - .ok_or_else(|| Trap::TableOutOfBounds)?; - - let store_id = store.id(); - let table = { - let instance = store.instance(context.instance); - instance.get_exported_table(store_id, segment.table_index) - }; - - if end > table.size_(&store) { - return Err(Trap::TableOutOfBounds.into()); - } - - let positions = start..end; - - match &segment.elements { - TableSegmentElements::Functions(funcs) => { - for (i, func_idx) in positions.zip(funcs) { - let func = { - let (instance, registry) = - store.instance_and_module_registry_mut(context.instance); - // SAFETY: the `store_id` passed to `get_exported_func` is - // indeed the store that owns the function. - unsafe { instance.get_exported_func(registry, store_id, *func_idx) } - }; - table.set_(&mut store, i, func.into())?; - } - } - TableSegmentElements::Expressions { exprs, ty: _ } => { - for (i, expr) in positions.zip(exprs) { - let val = const_evaluator - .eval(&mut store, limiter.as_deref_mut(), context, expr) - .await?; - table.set_(&mut store, i, val.ref_().unwrap())?; - } - } - } - } - - Ok(()) -} - -fn get_index(val: &Val, ty: wasmtime_environ::IndexType) -> u64 { - match ty { - wasmtime_environ::IndexType::I32 => val.unwrap_i32().cast_unsigned().into(), - wasmtime_environ::IndexType::I64 => val.unwrap_i64().cast_unsigned(), - } -} - -fn get_memory_init_start( - store: &mut StoreOpaque, - init: &MemoryInitializer, - instance: InstanceId, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, -) -> Result { - let mut store = OpaqueRootScope::new(store); - const_evaluator - .eval_int(&mut store, context, &init.offset) - .map(|v| { - get_index( - v, - store.instance(instance).env_module().memories[init.memory_index].idx_type, - ) - }) -} - -fn check_memory_init_bounds( - store: &mut StoreOpaque, - instance: InstanceId, - initializers: &[MemoryInitializer], - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, -) -> Result<()> { - for init in initializers { - let memory = store.instance_mut(instance).get_memory(init.memory_index); - let start = get_memory_init_start(store, init, instance, context, const_evaluator)?; - let end = usize::try_from(start) - .ok() - .and_then(|start| start.checked_add(init.data.len())); - - match end { - Some(end) if end <= memory.current_length() => { - // Initializer is in bounds - } - _ => { - bail!(Trap::MemoryOutOfBounds); - } - } - } - - Ok(()) -} - -fn initialize_memories( - store: &mut StoreOpaque, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - // Delegates to the `init_memory` method which is sort of a duplicate of - // `instance.memory_init_segment` but is used at compile-time in other - // contexts so is shared here to have only one method of memory - // initialization. - // - // This call to `init_memory` notably implements all the bells and whistles - // so errors only happen if an out-of-bounds segment is found, in which case - // a trap is returned. - - struct InitMemoryAtInstantiation<'a> { - module: &'a Module, - store: &'a mut StoreOpaque, - context: &'a mut ConstEvalContext, - const_evaluator: &'a mut ConstExprEvaluator, - } - - impl InitMemory for InitMemoryAtInstantiation<'_> { - fn memory_size_in_bytes( - &mut self, - memory: wasmtime_environ::MemoryIndex, - ) -> Result { - let len = self - .store - .instance(self.context.instance) - .get_memory(memory) - .current_length(); - let len = u64::try_from(len).unwrap(); - Ok(len) - } - - fn eval_offset( - &mut self, - memory: wasmtime_environ::MemoryIndex, - expr: &wasmtime_environ::ConstExpr, - ) -> Option { - let mut store = OpaqueRootScope::new(&mut *self.store); - let val = self - .const_evaluator - .eval_int(&mut store, self.context, expr) - .ok()?; - Some(get_index( - val, - store.instance(self.context.instance).env_module().memories[memory].idx_type, - )) - } - - fn write( - &mut self, - memory_index: wasmtime_environ::MemoryIndex, - init: &wasmtime_environ::StaticMemoryInitializer, - ) -> bool { - // If this initializer applies to a defined memory but that memory - // doesn't need initialization, due to something like copy-on-write - // pre-initializing it via mmap magic, then this initializer can be - // skipped entirely. - let instance = self.store.instance_mut(self.context.instance); - if let Some(memory_index) = self.module.defined_memory_index(memory_index) { - if !instance.memories[memory_index].1.needs_init() { - return true; - } - } - let memory = instance.get_memory(memory_index); - - unsafe { - let src = instance.wasm_data(init.data.clone()); - let offset = usize::try_from(init.offset).unwrap(); - let dst = memory.base.as_ptr().add(offset); - - assert!(offset + src.len() <= memory.current_length()); - - // FIXME audit whether this is safe in the presence of shared - // memory - // (https://github.com/bytecodealliance/wasmtime/issues/4203). - ptr::copy_nonoverlapping(src.as_ptr(), dst, src.len()) - } - true - } - } - - let ok = module - .memory_initialization - .init_memory(&mut InitMemoryAtInstantiation { - module, - store, - context, - const_evaluator, - }); - if !ok { - return Err(Trap::MemoryOutOfBounds.into()); - } - - Ok(()) -} - -fn check_init_bounds( - store: &mut StoreOpaque, - instance: InstanceId, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - check_table_init_bounds(store, instance, module, context, const_evaluator)?; - - match &module.memory_initialization { - MemoryInitialization::Segmented(initializers) => { - check_memory_init_bounds(store, instance, initializers, context, const_evaluator)?; - } - // Statically validated already to have everything in-bounds. - MemoryInitialization::Static { .. } => {} - } - - Ok(()) -} - -async fn initialize_globals( - store: &mut StoreOpaque, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - assert!(core::ptr::eq( - &**store.instance(context.instance).env_module(), - module - )); - - let mut store = OpaqueRootScope::new(store); - - for (index, init) in module.global_initializers.iter() { - // Attempt a simple, synchronous evaluation before hitting the - // general-purpose `.await` point below. This benchmarks ~15% faster in - // instantiation vs just falling through to `.await` below. - let val = if let Some(val) = const_evaluator.try_simple(init) { - val - } else { - const_evaluator - .eval(&mut store, limiter.as_deref_mut(), context, init) - .await? - }; - - let id = store.id(); - let index = module.global_index(index); - let mut instance = store.instance_mut(context.instance); - - #[cfg(feature = "wmemcheck")] - if index.as_u32() == 0 - && module.globals[index].wasm_ty == wasmtime_environ::WasmValType::I32 - { - if let Some(wmemcheck) = instance.as_mut().wmemcheck_state_mut() { - let size = usize::try_from(val.unwrap_i32()).unwrap(); - wmemcheck.set_stack_size(size); - } - } - - let global = instance.as_mut().get_exported_global(id, index); - - // Note that mutability is bypassed here because this is, by definition, - // initialization of globals meaning that if it's an immutable global - // this is the one and only write. - // - // SAFETY: this is a valid module so `val` should have the correct type - // for this global, and it's safe to write to a global for the first - // time as-is happening here. - unsafe { - global.set_unchecked(&mut store, &val)?; - } - } - Ok(()) -} - -async fn initialize_passive_elements( - store: &mut StoreOpaque, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - context: &mut ConstEvalContext, - const_evaluator: &mut ConstExprEvaluator, - module: &Module, -) -> Result<()> { - let store_id = store.id(); - - let instance = store.instance_mut(context.instance); - debug_assert!(instance.passive_elements.is_empty()); - instance - .passive_elements_mut() - .reserve(module.passive_elements.len())?; - - for (idx, segment) in &module.passive_elements { - match segment { - TableSegmentElements::Functions(func_indices) => { - let mut segment = - PassiveElementSegment::new(WasmRefType::FUNCREF, func_indices.len())?; - for func_idx in func_indices { - let (instance, registry) = - store.instance_and_module_registry_mut(context.instance); - // SAFETY: `store_id` is for the store that owns this instance. - let func = unsafe { instance.get_exported_func(registry, store_id, *func_idx) }; - segment.push(store, func.into())?; - } - let instance = store.instance_mut(context.instance); - debug_assert_eq!(instance.passive_elements.len(), idx.index()); - instance.passive_elements_mut().push(segment)?; - } - TableSegmentElements::Expressions { ty, exprs } => { - let mut segment = PassiveElementSegment::new(*ty, exprs.len())?; - for expr in exprs { - let mut store = OpaqueRootScope::new(&mut *store); - let val = const_evaluator - .eval(&mut store, limiter.as_deref_mut(), context, expr) - .await?; - segment.push(&mut store, *val)?; - } - let instance = store.instance_mut(context.instance); - debug_assert_eq!(instance.passive_elements.len(), idx.index()); - instance.passive_elements_mut().push(segment)?; - } - } - } - - debug_assert_eq!( - module.passive_elements.len(), - store.instance(context.instance).passive_elements.len() - ); - Ok(()) -} - -pub async fn initialize_instance( - store: &mut StoreOpaque, - mut limiter: Option<&mut StoreResourceLimiter<'_>>, - instance: InstanceId, - module: &Module, - is_bulk_memory: bool, - asyncness: Asyncness, -) -> Result<()> { - let mut context = ConstEvalContext::new(instance, asyncness); - let mut const_evaluator = ConstExprEvaluator::default(); - - // If bulk memory is not enabled, bounds check the data and element segments before - // making any changes. With bulk memory enabled, initializers are processed - // in-order and side effects are observed up to the point of an out-of-bounds - // initializer, so the early checking is not desired. - if !is_bulk_memory { - check_init_bounds(store, instance, &mut context, &mut const_evaluator, module)?; - } - - initialize_globals( - store, - limiter.as_deref_mut(), - &mut context, - &mut const_evaluator, - module, - ) - .await?; - - initialize_tables( - store, - limiter.as_deref_mut(), - &mut context, - &mut const_evaluator, - module, - ) - .await?; - - initialize_memories(store, &mut context, &mut const_evaluator, &module)?; - - if is_bulk_memory { - initialize_passive_elements(store, limiter, &mut context, &mut const_evaluator, &module) - .await?; - } - - Ok(()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 241057fa7128..f8641c87d6df 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -341,8 +341,7 @@ fn passive_elem_segment_base( store .instance_mut(instance) .passive_element_segment(elem_index) - .as_ptr() - .cast_mut() + .as_mut_ptr() .cast() } diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index ab097b978eb6..83844fb4eeff 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -4,7 +4,6 @@ mod vm_host_func_context; pub use self::vm_host_func_context::VMArrayCallHostFuncContext; -use crate::bail_bug; use crate::prelude::*; use crate::runtime::vm::{InterpreterRef, VMGcRef, VmPtr, VmSafe, f32x4, f64x2, i8x16}; use crate::store::StoreOpaque; @@ -20,7 +19,6 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use wasmtime_environ::{ BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, NUM_COMPONENT_CONTEXT_SLOTS, VMCONTEXT_MAGIC, VMSharedTypeIndex, - WasmHeapTopType, WasmValType, }; /// A function pointer that exposes the array calling convention. @@ -600,86 +598,6 @@ impl VMGlobalDefinition { Self { storage: [0; 16] } } - /// Create a `VMGlobalDefinition` from a `ValRaw`. - /// - /// # Unsafety - /// - /// This raw value's type must match the given `WasmValType`. - pub unsafe fn from_val_raw( - store: &mut StoreOpaque, - wasm_ty: WasmValType, - raw: ValRaw, - ) -> Result { - let mut global = Self::new(); - unsafe { - match wasm_ty { - WasmValType::I32 => *global.as_i32_mut() = raw.get_i32(), - WasmValType::I64 => *global.as_i64_mut() = raw.get_i64(), - WasmValType::F32 => *global.as_f32_bits_mut() = raw.get_f32(), - WasmValType::F64 => *global.as_f64_bits_mut() = raw.get_f64(), - WasmValType::V128 => global.set_u128(raw.get_v128()), - WasmValType::Ref(r) => match r.heap_type.top() { - WasmHeapTopType::Extern => { - let r = VMGcRef::from_raw_u32(raw.get_externref()); - global.init_gc_ref(store, r.as_ref())? - } - WasmHeapTopType::Any => { - let r = VMGcRef::from_raw_u32(raw.get_anyref()); - global.init_gc_ref(store, r.as_ref())? - } - WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(), - WasmHeapTopType::Cont => *global.as_func_ref_mut() = raw.get_funcref().cast(), // TODO(#10248): temporary hack. - WasmHeapTopType::Exn => { - let r = VMGcRef::from_raw_u32(raw.get_exnref()); - global.init_gc_ref(store, r.as_ref())? - } - }, - } - } - Ok(global) - } - - /// Get this global's value as a `ValRaw`. - /// - /// # Unsafety - /// - /// This global's value's type must match the given `WasmValType`. - pub unsafe fn to_val_raw( - &self, - store: &mut StoreOpaque, - wasm_ty: WasmValType, - ) -> Result { - unsafe { - Ok(match wasm_ty { - WasmValType::I32 => ValRaw::i32(*self.as_i32()), - WasmValType::I64 => ValRaw::i64(*self.as_i64()), - WasmValType::F32 => ValRaw::f32(*self.as_f32_bits()), - WasmValType::F64 => ValRaw::f64(*self.as_f64_bits()), - WasmValType::V128 => ValRaw::v128(self.get_u128()), - WasmValType::Ref(r) => match r.heap_type.top() { - WasmHeapTopType::Extern => ValRaw::externref(match self.as_gc_ref() { - Some(r) => store.clone_gc_ref(r).as_raw_u32(), - None => 0, - }), - WasmHeapTopType::Any => ValRaw::anyref({ - match self.as_gc_ref() { - Some(r) => store.clone_gc_ref(r).as_raw_u32(), - None => 0, - } - }), - WasmHeapTopType::Exn => ValRaw::exnref({ - match self.as_gc_ref() { - Some(r) => store.clone_gc_ref(r).as_raw_u32(), - None => 0, - } - }), - WasmHeapTopType::Func => ValRaw::funcref(self.as_func_ref().cast()), - WasmHeapTopType::Cont => bail_bug!("unimplemented"), // FIXME: #10248 stack switching support. - }, - }) - } - } - /// Return a reference to the value as an i32. pub unsafe fn as_i32(&self) -> &i32 { unsafe { &*(self.storage.as_ref().as_ptr().cast::()) } diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rs index 3939c26ff777..6a9a7f29828c 100644 --- a/crates/winch/src/compiler.rs +++ b/crates/winch/src/compiler.rs @@ -167,25 +167,15 @@ impl wasmtime_environ::Compiler for Compiler { }) } - fn compile_array_to_wasm_trampoline( - &self, - translation: &ModuleTranslation<'_>, - types: &ModuleTypesBuilder, - key: FuncKey, - symbol: &str, - ) -> Result { - self.trampolines - .compile_array_to_wasm_trampoline(translation, types, key, symbol) - } - - fn compile_wasm_to_array_trampoline( + fn compile_trampoline( &self, - wasm_func_ty: &wasmtime_environ::WasmFuncType, + translation: Option<&ModuleTranslation<'_>>, key: FuncKey, + types: &ModuleTypesBuilder, symbol: &str, ) -> Result { self.trampolines - .compile_wasm_to_array_trampoline(wasm_func_ty, key, symbol) + .compile_trampoline(translation, key, types, symbol) } fn append_code( @@ -236,14 +226,6 @@ impl wasmtime_environ::Compiler for Compiler { self.isa.create_systemv_cie() } - fn compile_wasm_to_builtin( - &self, - key: FuncKey, - symbol: &str, - ) -> Result { - self.trampolines.compile_wasm_to_builtin(key, symbol) - } - fn compiled_function_relocation_targets<'a>( &'a self, func: &'a dyn Any, @@ -280,45 +262,14 @@ impl wasmtime_environ::Compiler for NoInlineCompiler { Ok(body) } - fn compile_array_to_wasm_trampoline( - &self, - translation: &ModuleTranslation<'_>, - types: &ModuleTypesBuilder, - key: FuncKey, - symbol: &str, - ) -> Result { - let mut body = self - .0 - .compile_array_to_wasm_trampoline(translation, types, key, symbol)?; - if let Some(c) = self.0.inlining_compiler() { - c.finish_compiling(&mut body, None, symbol) - .map_err(|e| CompileError::Codegen(e.to_string()))?; - } - Ok(body) - } - - fn compile_wasm_to_array_trampoline( - &self, - wasm_func_ty: &wasmtime_environ::WasmFuncType, - key: FuncKey, - symbol: &str, - ) -> Result { - let mut body = self - .0 - .compile_wasm_to_array_trampoline(wasm_func_ty, key, symbol)?; - if let Some(c) = self.0.inlining_compiler() { - c.finish_compiling(&mut body, None, symbol) - .map_err(|e| CompileError::Codegen(e.to_string()))?; - } - Ok(body) - } - - fn compile_wasm_to_builtin( + fn compile_trampoline( &self, + translation: Option<&ModuleTranslation<'_>>, key: FuncKey, + types: &ModuleTypesBuilder, symbol: &str, ) -> Result { - let mut body = self.0.compile_wasm_to_builtin(key, symbol)?; + let mut body = self.0.compile_trampoline(translation, key, types, symbol)?; if let Some(c) = self.0.inlining_compiler() { c.finish_compiling(&mut body, None, symbol) .map_err(|e| CompileError::Codegen(e.to_string()))?; @@ -381,7 +332,7 @@ impl wasmtime_environ::Compiler for NoInlineCompiler { #[cfg(feature = "component-model")] impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler { - fn compile_trampoline( + fn compile_component_trampoline( &self, component: &wasmtime_environ::component::ComponentTranslation, types: &wasmtime_environ::component::ComponentTypesBuilder, @@ -393,7 +344,7 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler { let mut body = self .0 .component_compiler() - .compile_trampoline(component, types, key, abi, tunables, symbol)?; + .compile_component_trampoline(component, types, key, abi, tunables, symbol)?; if let Some(c) = self.0.inlining_compiler() { c.finish_compiling(&mut body, None, symbol) .map_err(|e| CompileError::Codegen(e.to_string()))?; diff --git a/src/commands/objdump.rs b/src/commands/objdump.rs index a81dc2a8be5f..c85607b63459 100644 --- a/src/commands/objdump.rs +++ b/src/commands/objdump.rs @@ -205,6 +205,7 @@ impl ObjdumpCommand { || name.ends_with("_array_call") || name.ends_with("_wasm_call") || name.contains("unsafe-intrinsics-") + || name.contains("module_start") { Func::Trampoline } else if name.contains("libcall") || name.starts_with("component") { diff --git a/tests/all/arrays.rs b/tests/all/arrays.rs index 31f68b306e9a..b42a38855dd8 100644 --- a/tests/all/arrays.rs +++ b/tests/all/arrays.rs @@ -1002,6 +1002,7 @@ fn issue_13034_array_layout_overflow() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn host_arrayref_has_trace_info_for_gc() -> Result<()> { for collector in [Collector::Copying, Collector::DeferredReferenceCounting] { println!("Using GC collector: {collector:?}"); diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs index 96d45dcebd0f..a7f34fb7fcf8 100644 --- a/tests/all/cli_tests.rs +++ b/tests/all/cli_tests.rs @@ -3051,11 +3051,11 @@ fn profile_guest() -> Result<()> { None, )?; - assert!(output.status.success()); let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); println!("> stdout:\n{stdout}"); println!("> stderr:\n{stderr}"); + assert!(output.status.success()); assert!(!stderr.contains("Error")); let out_json = std::fs::read_to_string(format!("{dir}/out.json")).unwrap(); println!("> out.json:\n{out_json}"); diff --git a/tests/all/exceptions.rs b/tests/all/exceptions.rs index e4f326877a8c..cde1dec7424e 100644 --- a/tests/all/exceptions.rs +++ b/tests/all/exceptions.rs @@ -216,6 +216,7 @@ fn exception_across_no_wasm(config: &mut Config) -> Result<()> { } #[wasmtime_test(wasm_features(gc, exceptions))] +#[cfg_attr(miri, ignore)] fn gc_with_exnref_global(config: &mut Config) -> Result<()> { let engine = Engine::new(config)?; let mut store = Store::new(&engine, ()); diff --git a/tests/all/exnrefs.rs b/tests/all/exnrefs.rs index 6251d01ff669..de0b73efbba3 100644 --- a/tests/all/exnrefs.rs +++ b/tests/all/exnrefs.rs @@ -85,6 +85,7 @@ fn exn_objects() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn host_exnref_has_trace_info_for_gc() -> Result<()> { for collector in [Collector::Copying, Collector::DeferredReferenceCounting] { println!("Using GC collector: {collector:?}"); diff --git a/tests/all/fuel.wast b/tests/all/fuel.wast index 7196df646a94..f918eb94aca3 100644 --- a/tests/all/fuel.wast +++ b/tests/all/fuel.wast @@ -1,11 +1,11 @@ (assert_fuel 0 (module)) -(assert_fuel 1 +(assert_fuel 3 (module (func $f) (start $f))) -(assert_fuel 2 +(assert_fuel 4 (module (func $f i32.const 0 @@ -13,7 +13,7 @@ ) (start $f))) -(assert_fuel 1 +(assert_fuel 3 (module (func $f block @@ -21,14 +21,14 @@ ) (start $f))) -(assert_fuel 1 +(assert_fuel 3 (module (func $f unreachable ) (start $f))) -(assert_fuel 7 +(assert_fuel 9 (module (func $f i32.const 0 @@ -41,7 +41,7 @@ ) (start $f))) -(assert_fuel 1 +(assert_fuel 3 (module (func $f return @@ -55,7 +55,7 @@ ) (start $f))) -(assert_fuel 3 +(assert_fuel 5 (module (func $f i32.const 0 @@ -65,7 +65,7 @@ ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f i32.const 1 @@ -76,7 +76,7 @@ ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f i32.const 1 @@ -89,7 +89,7 @@ ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f i32.const 0 @@ -102,7 +102,7 @@ ) (start $f))) -(assert_fuel 3 +(assert_fuel 5 (module (func $f block @@ -114,7 +114,7 @@ ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f block @@ -127,7 +127,7 @@ (start $f))) ;; count code before unreachable -(assert_fuel 2 +(assert_fuel 4 (module (func $f i32.const 0 @@ -136,7 +136,7 @@ (start $f))) ;; count code before return -(assert_fuel 2 +(assert_fuel 4 (module (func $f i32.const 0 @@ -145,14 +145,14 @@ (start $f))) ;; cross-function fuel works -(assert_fuel 3 +(assert_fuel 5 (module (func $f call $other ) (func $other) (start $f))) -(assert_fuel 5 +(assert_fuel 7 (module (func $f i32.const 0 @@ -162,7 +162,7 @@ ) (func $other (param i32)) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f call $other @@ -172,7 +172,7 @@ i32.const 0 ) (start $f))) -(assert_fuel 4 +(assert_fuel 6 (module (func $f i32.const 0 @@ -183,14 +183,14 @@ (start $f))) ;; loops! -(assert_fuel 1 +(assert_fuel 3 (module (func $f loop end ) (start $f))) -(assert_fuel 53 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func +(assert_fuel 55 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func (module (func $f (local i32) @@ -207,7 +207,7 @@ ) (start $f))) -(assert_fuel 105 +(assert_fuel 107 (module (memory 1) (func $f @@ -218,7 +218,7 @@ ) (start $f))) -(assert_fuel 105 +(assert_fuel 107 (module (memory 1) (func $f @@ -229,7 +229,7 @@ ) (start $f))) -(assert_fuel 25 +(assert_fuel 27 (module (memory 1) (func $f @@ -241,7 +241,7 @@ (start $f) (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))) -(assert_fuel 105 +(assert_fuel 107 (module (table 100 funcref) (func $f @@ -252,7 +252,7 @@ ) (start $f))) -(assert_fuel 105 +(assert_fuel 107 (module (table 100 funcref) (func $f @@ -263,7 +263,7 @@ ) (start $f))) -(assert_fuel 104 +(assert_fuel 106 (module (table 0 funcref) (func $f @@ -274,7 +274,7 @@ ) (start $f))) -(assert_fuel 25 +(assert_fuel 27 (module (table 20 funcref) (func $f @@ -286,7 +286,7 @@ (start $f) (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f))) -(assert_fuel 107 +(assert_fuel 211 (module (type $a (array (mut i8))) (global $a (ref $a) (array.new_default $a (i32.const 100))) @@ -300,7 +300,7 @@ ) (start $f))) -(assert_fuel 106 +(assert_fuel 210 (module (type $a (array (mut i8))) (global $a (ref $a) (array.new_default $a (i32.const 100))) @@ -313,7 +313,7 @@ ) (start $f))) -(assert_fuel 24 +(assert_fuel 26 (module (type $a (array (mut i8))) (func $f @@ -325,7 +325,7 @@ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") (start $f))) -(assert_fuel 26 +(assert_fuel 130 (module (type $a (array (mut i8))) (global $a (ref $a) (array.new_default $a (i32.const 100))) @@ -339,7 +339,7 @@ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") (start $f))) -(assert_fuel 24 +(assert_fuel 26 (module (type $a (array (mut funcref))) (func $f @@ -351,7 +351,7 @@ (start $f) (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f))) -(assert_fuel 26 +(assert_fuel 130 (module (type $a (array (mut funcref))) (global $a (ref $a) (array.new_default $a (i32.const 100))) @@ -365,7 +365,7 @@ (start $f) (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f))) -(assert_fuel 403 ;; 400 bytes set + 1 func + 2 instructions (drop is 0) +(assert_fuel 105 (module (type $a (array (mut funcref))) (func $f @@ -375,7 +375,7 @@ ) (start $f))) -(assert_fuel 404 ;; 400 bytes set + 1 func + 3 instructions (drop is 0) +(assert_fuel 106 (module (type $a (array (mut funcref))) (func $f diff --git a/tests/all/gc.rs b/tests/all/gc.rs index 7301fe6bf2c8..b86a73f0b4ae 100644 --- a/tests/all/gc.rs +++ b/tests/all/gc.rs @@ -373,6 +373,7 @@ fn table_drops_externref() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn global_init_no_leak() -> Result<()> { let (mut store, module) = ref_types_module( false, @@ -1157,6 +1158,7 @@ fn issue_9669() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn drc_transitive_drop_cons_list() -> Result<()> { let _ = env_logger::try_init(); @@ -3315,6 +3317,8 @@ fn typed_option_noextern() -> Result<()> { /// A test that performs a GC without actually compiling or running any Wasm /// functions so we can run this test under MIRI. #[test] +// FIXME: need to fold this into other miri testing which precompiles +#[cfg_attr(miri, ignore)] fn miri_gc_smoke_test() -> Result<()> { for collector in [ Collector::Copying, diff --git a/tests/all/memory.rs b/tests/all/memory.rs index 8897ad081d70..f321f5d98c36 100644 --- a/tests/all/memory.rs +++ b/tests/all/memory.rs @@ -510,6 +510,7 @@ fn dynamic_extra_growth_unchanged_pointer(config: &mut Config) -> Result<()> { // determining this failure we shouldn't hit any overflows or anything like that // (checked via debug-mode tests). #[wasmtime_test] +#[cfg_attr(miri, ignore)] fn memory64_maximum_minimum(config: &mut Config) -> Result<()> { config.wasm_memory64(true); let engine = Engine::new(&config)?; diff --git a/tests/all/module.rs b/tests/all/module.rs index da99f7442b70..6a4daff1a78b 100644 --- a/tests/all/module.rs +++ b/tests/all/module.rs @@ -141,6 +141,7 @@ fn serialize_deterministic() { // into an initialization image doesn't unnecessarily create a massive module by // accident with a very large initialization image in it. #[test] +#[cfg_attr(miri, ignore)] fn serialize_not_overly_massive() -> Result<()> { let mut config = Config::new(); config.memory_guaranteed_dense_image_size(1 << 20); diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs index c14a6dbbc587..ffb7482003ca 100644 --- a/tests/all/pooling_allocator.rs +++ b/tests/all/pooling_allocator.rs @@ -98,6 +98,7 @@ fn memory_limit() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn memory_init() -> Result<()> { let mut pool = crate::small_pool_config(); pool.max_memory_size(2 << 16).table_elements(0); @@ -441,6 +442,7 @@ fn total_core_instances_limit() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn preserve_data_segments() -> Result<()> { let mut pool = crate::small_pool_config(); pool.total_memories(2); @@ -516,6 +518,7 @@ fn multi_memory_with_imported_memories() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn drop_externref_global_during_module_init() -> Result<()> { struct Limiter; @@ -1248,6 +1251,7 @@ fn decommit_batching() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn tricky_empty_table_with_empty_virtual_memory_alloc() -> Result<()> { // Configure the pooling allocator to have no access to virtual memory, e.g. // no table elements but a single table. This should technically support a @@ -1458,6 +1462,7 @@ fn pooling_reuse_resets() -> Result<()> { // asserts that the previous image is indeed not available any more as that // would otherwise mean data was leaked between modules. #[test] +#[cfg_attr(miri, ignore)] fn memory_reset_if_instantiation_fails() -> Result<()> { struct Limiter; diff --git a/tests/all/structs.rs b/tests/all/structs.rs index 59e14ae087db..652f028f31d0 100644 --- a/tests/all/structs.rs +++ b/tests/all/structs.rs @@ -912,6 +912,7 @@ fn issue_9714(config: &mut Config) -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn host_structref_has_trace_info_for_gc() -> Result<()> { for collector in [Collector::Copying, Collector::DeferredReferenceCounting] { println!("Using GC collector: {collector:?}"); diff --git a/tests/all/table.rs b/tests/all/table.rs index b4375c479a46..6d8d9bd843ff 100644 --- a/tests/all/table.rs +++ b/tests/all/table.rs @@ -368,6 +368,7 @@ fn host_table_keep_type_registration() -> Result<()> { } #[test] +#[cfg_attr(miri, ignore)] fn gc_store_with_table_initializers() -> Result<()> { let mut config = Config::new(); config.wasm_gc(true); diff --git a/tests/disas/array-fill-anyref.wat b/tests/disas/array-fill-anyref.wat index 52bc52418829..428cb320f919 100644 --- a/tests/disas/array-fill-anyref.wat +++ b/tests/disas/array-fill-anyref.wat @@ -81,14 +81,12 @@ ;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:2 sig0 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): ;; @003e trapz v2, user16 -;; @003e v44 = load.i64 notrap aligned readonly can_move v0+8 -;; @003e v7 = load.i64 notrap aligned readonly can_move v44+32 +;; @003e v49 = load.i64 notrap aligned readonly can_move v0+8 +;; @003e v7 = load.i64 notrap aligned readonly can_move v49+32 ;; @003e v6 = uextend.i64 v2 ;; @003e v8 = iadd v7, v6 ;; @003e v9 = iconst.i64 16 @@ -100,19 +98,33 @@ ;; @003e v12 = uextend.i64 v11 ;; @003e v17 = icmp ugt v16, v12 ;; @003e trapnz v17, user17 -;; @003e v28 = load.i64 notrap aligned v44+40 -;; v40 = iconst.i64 20 -;; @003e v21 = iadd v8, v40 ; v40 = 20 -;; v48 = iconst.i64 2 -;; v49 = ishl v13, v48 ; v48 = 2 -;; @003e v24 = iadd v21, v49 -;; v51 = ishl v14, v48 ; v48 = 2 -;; @003e v30 = uadd_overflow_trap v24, v51, user2 +;; @003e v28 = load.i64 notrap aligned v49+40 +;; v45 = iconst.i64 20 +;; @003e v21 = iadd v8, v45 ; v45 = 20 +;; v53 = iconst.i64 2 +;; v54 = ishl v13, v53 ; v53 = 2 +;; @003e v24 = iadd v21, v54 +;; v56 = ishl v14, v53 ; v53 = 2 +;; @003e v30 = uadd_overflow_trap v24, v56, user2 ;; @003e v29 = iadd v7, v28 ;; @003e v31 = icmp ugt v30, v29 ;; @003e trapnz v31, user2 +;; v51 = iconst.i64 0 +;; @003e v33 = icmp eq v14, v51 ; v51 = 0 ;; @003a v5 = iconst.i32 0 -;; @003e call fn0(v0, v24, v5, v51) ; v5 = 0 +;; v44 = iconst.i64 4 +;; @003e v32 = iadd v24, v56 +;; @003e brif v33, block3, block2(v24) +;; +;; block2(v34: i64): +;; v58 = iconst.i32 0 +;; @003e store user2 little v58, v34 ; v58 = 0 +;; v59 = iconst.i64 4 +;; v60 = iadd v34, v59 ; v59 = 4 +;; @003e v36 = icmp eq v60, v32 +;; @003e brif v36, block3, block2(v60) +;; +;; block3: ;; @0041 jump block1 ;; ;; block1: diff --git a/tests/disas/array-fill-externref.wat b/tests/disas/array-fill-externref.wat index a327322edef5..542857bb457a 100644 --- a/tests/disas/array-fill-externref.wat +++ b/tests/disas/array-fill-externref.wat @@ -77,14 +77,12 @@ ;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:2 sig0 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): ;; @003d trapz v2, user16 -;; @003d v44 = load.i64 notrap aligned readonly can_move v0+8 -;; @003d v7 = load.i64 notrap aligned readonly can_move v44+32 +;; @003d v49 = load.i64 notrap aligned readonly can_move v0+8 +;; @003d v7 = load.i64 notrap aligned readonly can_move v49+32 ;; @003d v6 = uextend.i64 v2 ;; @003d v8 = iadd v7, v6 ;; @003d v9 = iconst.i64 16 @@ -96,19 +94,33 @@ ;; @003d v12 = uextend.i64 v11 ;; @003d v17 = icmp ugt v16, v12 ;; @003d trapnz v17, user17 -;; @003d v28 = load.i64 notrap aligned v44+40 -;; v40 = iconst.i64 20 -;; @003d v21 = iadd v8, v40 ; v40 = 20 -;; v48 = iconst.i64 2 -;; v49 = ishl v13, v48 ; v48 = 2 -;; @003d v24 = iadd v21, v49 -;; v51 = ishl v14, v48 ; v48 = 2 -;; @003d v30 = uadd_overflow_trap v24, v51, user2 +;; @003d v28 = load.i64 notrap aligned v49+40 +;; v45 = iconst.i64 20 +;; @003d v21 = iadd v8, v45 ; v45 = 20 +;; v53 = iconst.i64 2 +;; v54 = ishl v13, v53 ; v53 = 2 +;; @003d v24 = iadd v21, v54 +;; v56 = ishl v14, v53 ; v53 = 2 +;; @003d v30 = uadd_overflow_trap v24, v56, user2 ;; @003d v29 = iadd v7, v28 ;; @003d v31 = icmp ugt v30, v29 ;; @003d trapnz v31, user2 +;; v51 = iconst.i64 0 +;; @003d v33 = icmp eq v14, v51 ; v51 = 0 ;; @0039 v5 = iconst.i32 0 -;; @003d call fn0(v0, v24, v5, v51) ; v5 = 0 +;; v44 = iconst.i64 4 +;; @003d v32 = iadd v24, v56 +;; @003d brif v33, block3, block2(v24) +;; +;; block2(v34: i64): +;; v58 = iconst.i32 0 +;; @003d store user2 little v58, v34 ; v58 = 0 +;; v59 = iconst.i64 4 +;; v60 = iadd v34, v59 ; v59 = 4 +;; @003d v36 = icmp eq v60, v32 +;; @003d brif v36, block3, block2(v60) +;; +;; block3: ;; @0040 jump block1 ;; ;; block1: diff --git a/tests/disas/array-fill-funcref.wat b/tests/disas/array-fill-funcref.wat index 4f594096bb1c..026b75494258 100644 --- a/tests/disas/array-fill-funcref.wat +++ b/tests/disas/array-fill-funcref.wat @@ -88,14 +88,14 @@ ;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:2 sig0 +;; sig0 = (i64 vmctx, i64) -> i64 tail +;; fn0 = colocated u805306368:25 sig0 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): ;; @0049 trapz v2, user16 -;; @0049 v44 = load.i64 notrap aligned readonly can_move v0+8 -;; @0049 v7 = load.i64 notrap aligned readonly can_move v44+32 +;; @0049 v52 = load.i64 notrap aligned readonly can_move v0+8 +;; @0049 v7 = load.i64 notrap aligned readonly can_move v52+32 ;; @0049 v6 = uextend.i64 v2 ;; @0049 v8 = iadd v7, v6 ;; @0049 v9 = iconst.i64 16 @@ -107,19 +107,34 @@ ;; @0049 v12 = uextend.i64 v11 ;; @0049 v17 = icmp ugt v16, v12 ;; @0049 trapnz v17, user17 -;; @0049 v28 = load.i64 notrap aligned v44+40 -;; v40 = iconst.i64 20 -;; @0049 v21 = iadd v8, v40 ; v40 = 20 -;; v47 = iconst.i64 2 -;; v48 = ishl v13, v47 ; v47 = 2 -;; @0049 v24 = iadd v21, v48 -;; v50 = ishl v14, v47 ; v47 = 2 -;; @0049 v30 = uadd_overflow_trap v24, v50, user2 +;; @0049 v28 = load.i64 notrap aligned v52+40 +;; v48 = iconst.i64 20 +;; @0049 v21 = iadd v8, v48 ; v48 = 20 +;; v55 = iconst.i64 2 +;; v56 = ishl v13, v55 ; v55 = 2 +;; @0049 v24 = iadd v21, v56 +;; v58 = ishl v14, v55 ; v55 = 2 +;; @0049 v30 = uadd_overflow_trap v24, v58, user2 ;; @0049 v29 = iadd v7, v28 ;; @0049 v31 = icmp ugt v30, v29 ;; @0049 trapnz v31, user2 -;; @0049 v32 = iconst.i32 0 -;; @0049 call fn0(v0, v24, v32, v50) ; v32 = 0 +;; @0045 v5 = iconst.i64 0 +;; @0049 v33 = icmp eq v14, v5 ; v5 = 0 +;; v47 = iconst.i64 4 +;; @0049 v32 = iadd v24, v58 +;; @0049 brif v33, block3, block2(v24) +;; +;; block2(v34: i64): +;; v60 = iconst.i64 0 +;; @0049 v36 = call fn0(v0, v60) ; v60 = 0 +;; @0049 v37 = ireduce.i32 v36 +;; @0049 store user2 little v37, v34 +;; v61 = iconst.i64 4 +;; v62 = iadd v34, v61 ; v61 = 4 +;; @0049 v39 = icmp eq v62, v32 +;; @0049 brif v39, block3, block2(v62) +;; +;; block3: ;; @004c jump block1 ;; ;; block1: diff --git a/tests/disas/array-fill-i31ref.wat b/tests/disas/array-fill-i31ref.wat index 6723ac376d17..b094bdde3f99 100644 --- a/tests/disas/array-fill-i31ref.wat +++ b/tests/disas/array-fill-i31ref.wat @@ -81,14 +81,12 @@ ;; gv4 = load.i64 notrap aligned readonly can_move gv3+8 ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 -;; sig0 = (i64 vmctx, i64, i32, i64) tail -;; fn0 = colocated u805306368:2 sig0 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32): ;; @003e trapz v2, user16 -;; @003e v44 = load.i64 notrap aligned readonly can_move v0+8 -;; @003e v7 = load.i64 notrap aligned readonly can_move v44+32 +;; @003e v49 = load.i64 notrap aligned readonly can_move v0+8 +;; @003e v7 = load.i64 notrap aligned readonly can_move v49+32 ;; @003e v6 = uextend.i64 v2 ;; @003e v8 = iadd v7, v6 ;; @003e v9 = iconst.i64 16 @@ -100,19 +98,33 @@ ;; @003e v12 = uextend.i64 v11 ;; @003e v17 = icmp ugt v16, v12 ;; @003e trapnz v17, user17 -;; @003e v28 = load.i64 notrap aligned v44+40 -;; v40 = iconst.i64 20 -;; @003e v21 = iadd v8, v40 ; v40 = 20 -;; v48 = iconst.i64 2 -;; v49 = ishl v13, v48 ; v48 = 2 -;; @003e v24 = iadd v21, v49 -;; v51 = ishl v14, v48 ; v48 = 2 -;; @003e v30 = uadd_overflow_trap v24, v51, user2 +;; @003e v28 = load.i64 notrap aligned v49+40 +;; v45 = iconst.i64 20 +;; @003e v21 = iadd v8, v45 ; v45 = 20 +;; v53 = iconst.i64 2 +;; v54 = ishl v13, v53 ; v53 = 2 +;; @003e v24 = iadd v21, v54 +;; v56 = ishl v14, v53 ; v53 = 2 +;; @003e v30 = uadd_overflow_trap v24, v56, user2 ;; @003e v29 = iadd v7, v28 ;; @003e v31 = icmp ugt v30, v29 ;; @003e trapnz v31, user2 +;; v51 = iconst.i64 0 +;; @003e v33 = icmp eq v14, v51 ; v51 = 0 ;; @003a v5 = iconst.i32 0 -;; @003e call fn0(v0, v24, v5, v51) ; v5 = 0 +;; v44 = iconst.i64 4 +;; @003e v32 = iadd v24, v56 +;; @003e brif v33, block3, block2(v24) +;; +;; block2(v34: i64): +;; v58 = iconst.i32 0 +;; @003e store user2 little v58, v34 ; v58 = 0 +;; v59 = iconst.i64 4 +;; v60 = iadd v34, v59 ; v59 = 4 +;; @003e v36 = icmp eq v60, v32 +;; @003e brif v36, block3, block2(v60) +;; +;; block3: ;; @0041 jump block1 ;; ;; block1: diff --git a/tests/disas/gc/array-init-data.wat b/tests/disas/gc/array-init-data.wat index 39ff89c7984d..c44bdab873d9 100644 --- a/tests/disas/gc/array-init-data.wat +++ b/tests/disas/gc/array-init-data.wat @@ -27,8 +27,8 @@ ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32): ;; @002a trapz v2, user16 -;; @002a v57 = load.i64 notrap aligned readonly can_move v0+8 -;; @002a v7 = load.i64 notrap aligned readonly can_move v57+32 +;; @002a v58 = load.i64 notrap aligned readonly can_move v0+8 +;; @002a v7 = load.i64 notrap aligned readonly can_move v58+32 ;; @002a v6 = uextend.i64 v2 ;; @002a v8 = iadd v7, v6 ;; @002a v9 = iconst.i64 16 @@ -40,22 +40,23 @@ ;; @002a v12 = uextend.i64 v11 ;; @002a v17 = icmp ugt v16, v12 ;; @002a trapnz v17, user17 -;; @002a v26 = uload32 notrap aligned v0+56 -;; @002a v27 = uextend.i64 v4 -;; @002a v30 = iadd v27, v14 -;; @002a v31 = icmp ugt v30, v26 -;; @002a trapnz v31, heap_oob -;; @002a v33 = load.i64 notrap aligned v0+48 -;; @002a v40 = load.i64 notrap aligned v57+40 -;; v53 = iconst.i64 20 -;; @002a v21 = iadd v8, v53 ; v53 = 20 +;; @002a v26 = load.i32 notrap aligned v0+56 +;; @002a v28 = uextend.i64 v4 +;; @002a v31 = iadd v28, v14 +;; @002a v27 = uextend.i64 v26 +;; @002a v32 = icmp ugt v31, v27 +;; @002a trapnz v32, heap_oob +;; @002a v34 = load.i64 notrap aligned v0+48 +;; @002a v41 = load.i64 notrap aligned v58+40 +;; v54 = iconst.i64 20 +;; @002a v21 = iadd v8, v54 ; v54 = 20 ;; @002a v24 = iadd v21, v13 -;; @002a v42 = uadd_overflow_trap v24, v14, user2 -;; @002a v41 = iadd v7, v40 -;; @002a v43 = icmp ugt v42, v41 -;; @002a trapnz v43, user2 -;; @002a v35 = iadd v33, v27 -;; @002a call fn0(v0, v24, v35, v14) +;; @002a v43 = uadd_overflow_trap v24, v14, user2 +;; @002a v42 = iadd v7, v41 +;; @002a v44 = icmp ugt v43, v42 +;; @002a trapnz v44, user2 +;; @002a v36 = iadd v34, v28 +;; @002a call fn0(v0, v24, v36, v14) ;; @002e jump block1 ;; ;; block1: diff --git a/tests/disas/gc/array-new-data.wat b/tests/disas/gc/array-new-data.wat index 2ff85b8dfbfb..29e109b03e99 100644 --- a/tests/disas/gc/array-new-data.wat +++ b/tests/disas/gc/array-new-data.wat @@ -92,96 +92,98 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @0025 v6 = uload32 notrap aligned v0+56 -;; @0025 v7 = uextend.i64 v2 -;; @0025 v8 = uextend.i64 v3 -;; @0025 v10 = iadd v7, v8 -;; @0025 v11 = icmp ugt v10, v6 -;; @0025 trapnz v11, heap_oob -;; @0025 v13 = load.i64 notrap aligned v0+48 -;; v124 = iconst.i64 32 -;; @0025 v19 = ushr v8, v124 ; v124 = 32 -;; @0025 trapnz v19, user18 -;; @0025 v16 = iconst.i32 20 -;; @0025 v21 = uadd_overflow_trap v16, v3, user18 ; v16 = 20 -;; @0025 v23 = load.i64 notrap aligned readonly can_move v0+32 -;; @0025 v24 = load.i32 notrap aligned v23 -;; @0025 v25 = load.i32 notrap aligned v23+4 -;; @0025 v31 = uextend.i64 v24 -;; @0025 v26 = uextend.i64 v21 -;; @0025 v27 = iconst.i64 15 -;; @0025 v29 = iadd v26, v27 ; v27 = 15 -;; @0025 v28 = iconst.i64 -16 -;; @0025 v30 = band v29, v28 ; v28 = -16 -;; @0025 v32 = iadd v31, v30 -;; @0025 v33 = uextend.i64 v25 -;; @0025 v34 = icmp ule v32, v33 -;; @0025 brif v34, block2, block3 +;; @0025 v6 = load.i32 notrap aligned v0+56 +;; @0025 v8 = uextend.i64 v2 +;; @0025 v9 = uextend.i64 v3 +;; @0025 v11 = iadd v8, v9 +;; @0025 v7 = uextend.i64 v6 +;; @0025 v12 = icmp ugt v11, v7 +;; @0025 trapnz v12, heap_oob +;; @0025 v14 = load.i64 notrap aligned v0+48 +;; v126 = iconst.i64 32 +;; @0025 v20 = ushr v9, v126 ; v126 = 32 +;; @0025 trapnz v20, user18 +;; @0025 v17 = iconst.i32 20 +;; @0025 v22 = uadd_overflow_trap v17, v3, user18 ; v17 = 20 +;; @0025 v24 = load.i64 notrap aligned readonly can_move v0+32 +;; @0025 v25 = load.i32 notrap aligned v24 +;; @0025 v26 = load.i32 notrap aligned v24+4 +;; @0025 v32 = uextend.i64 v25 +;; @0025 v27 = uextend.i64 v22 +;; @0025 v28 = iconst.i64 15 +;; @0025 v30 = iadd v27, v28 ; v28 = 15 +;; @0025 v29 = iconst.i64 -16 +;; @0025 v31 = band v30, v29 ; v29 = -16 +;; @0025 v33 = iadd v32, v31 +;; @0025 v34 = uextend.i64 v26 +;; @0025 v35 = icmp ule v33, v34 +;; @0025 brif v35, block2, block3 ;; ;; block2: -;; v134 = iconst.i32 15 -;; v135 = iadd.i32 v21, v134 ; v134 = 15 -;; v138 = iconst.i32 -16 -;; v139 = band v135, v138 ; v138 = -16 -;; v141 = iadd.i32 v24, v139 -;; @0025 store notrap aligned region0 v141, v23 -;; v155 = iconst.i32 -1476395002 -;; v156 = load.i64 notrap aligned readonly can_move v0+8 -;; v157 = load.i64 notrap aligned readonly can_move v156+32 -;; @0025 v48 = iadd v157, v31 -;; @0025 store notrap aligned v155, v48 ; v155 = -1476395002 -;; v158 = load.i64 notrap aligned readonly can_move v0+40 -;; v159 = load.i32 notrap aligned readonly can_move v158 -;; @0025 store notrap aligned v159, v48+4 -;; v160 = band.i64 v29, v28 ; v28 = -16 -;; @0025 istore32 notrap aligned v160, v48+8 -;; @0025 jump block4(v24, v48) +;; v136 = iconst.i32 15 +;; v137 = iadd.i32 v22, v136 ; v136 = 15 +;; v140 = iconst.i32 -16 +;; v141 = band v137, v140 ; v140 = -16 +;; v143 = iadd.i32 v25, v141 +;; @0025 store notrap aligned region0 v143, v24 +;; v157 = iconst.i32 -1476395002 +;; v158 = load.i64 notrap aligned readonly can_move v0+8 +;; v159 = load.i64 notrap aligned readonly can_move v158+32 +;; @0025 v49 = iadd v159, v32 +;; @0025 store notrap aligned v157, v49 ; v157 = -1476395002 +;; v160 = load.i64 notrap aligned readonly can_move v0+40 +;; v161 = load.i32 notrap aligned readonly can_move v160 +;; @0025 store notrap aligned v161, v49+4 +;; v162 = band.i64 v30, v29 ; v29 = -16 +;; @0025 istore32 notrap aligned v162, v49+8 +;; @0025 jump block4(v25, v49) ;; ;; block3 cold: -;; @0025 v36 = iconst.i32 -1476395002 -;; @0025 v38 = load.i64 notrap aligned readonly can_move v0+40 -;; @0025 v39 = load.i32 notrap aligned readonly can_move v38 -;; @0025 v40 = iconst.i32 16 -;; @0025 v41 = call fn0(v0, v36, v39, v21, v40) ; v36 = -1476395002, v40 = 16 -;; @0025 v120 = load.i64 notrap aligned readonly can_move v0+8 -;; @0025 v42 = load.i64 notrap aligned readonly can_move v120+32 -;; @0025 v43 = uextend.i64 v41 -;; @0025 v44 = iadd v42, v43 -;; @0025 jump block4(v41, v44) +;; @0025 v37 = iconst.i32 -1476395002 +;; @0025 v39 = load.i64 notrap aligned readonly can_move v0+40 +;; @0025 v40 = load.i32 notrap aligned readonly can_move v39 +;; @0025 v41 = iconst.i32 16 +;; @0025 v42 = call fn0(v0, v37, v40, v22, v41) ; v37 = -1476395002, v41 = 16 +;; @0025 v122 = load.i64 notrap aligned readonly can_move v0+8 +;; @0025 v43 = load.i64 notrap aligned readonly can_move v122+32 +;; @0025 v44 = uextend.i64 v42 +;; @0025 v45 = iadd v43, v44 +;; @0025 jump block4(v42, v45) ;; -;; block4(v53: i32, v54: i64): -;; v119 = stack_addr.i64 ss0 -;; store notrap v53, v119 -;; v118 = iconst.i64 16 -;; @0025 v55 = iadd v54, v118 ; v118 = 16 -;; @0025 store.i32 user2 v3, v55 -;; v99 = load.i32 notrap v119 -;; @0025 trapz v99, user16 -;; v161 = load.i64 notrap aligned readonly can_move v0+8 -;; v162 = load.i64 notrap aligned readonly can_move v161+32 -;; @0025 v57 = uextend.i64 v99 -;; @0025 v59 = iadd v162, v57 -;; @0025 v61 = iadd v59, v118 ; v118 = 16 -;; @0025 v62 = load.i32 user2 readonly v61 -;; @0025 v63 = uextend.i64 v62 -;; @0025 v68 = icmp.i64 ugt v8, v63 -;; @0025 trapnz v68, user17 -;; @0025 v77 = uload32.i64 notrap aligned v0+56 -;; @0025 v82 = icmp.i64 ugt v10, v77 -;; @0025 trapnz v82, heap_oob -;; @0025 v84 = load.i64 notrap aligned v0+48 -;; @0025 v91 = load.i64 notrap aligned v161+40 -;; v109 = iconst.i64 20 -;; @0025 v72 = iadd v59, v109 ; v109 = 20 -;; @0025 v93 = uadd_overflow_trap v72, v8, user2 -;; @0025 v92 = iadd v162, v91 -;; @0025 v94 = icmp ugt v93, v92 -;; @0025 trapnz v94, user2 -;; @0025 v86 = iadd v84, v7 -;; @0025 call fn1(v0, v72, v86, v8), stack_map=[i32 @ ss0+0] -;; v96 = load.i32 notrap v119 +;; block4(v54: i32, v55: i64): +;; v121 = stack_addr.i64 ss0 +;; store notrap v54, v121 +;; v120 = iconst.i64 16 +;; @0025 v56 = iadd v55, v120 ; v120 = 16 +;; @0025 store.i32 user2 v3, v56 +;; v101 = load.i32 notrap v121 +;; @0025 trapz v101, user16 +;; v163 = load.i64 notrap aligned readonly can_move v0+8 +;; v164 = load.i64 notrap aligned readonly can_move v163+32 +;; @0025 v58 = uextend.i64 v101 +;; @0025 v60 = iadd v164, v58 +;; @0025 v62 = iadd v60, v120 ; v120 = 16 +;; @0025 v63 = load.i32 user2 readonly v62 +;; @0025 v64 = uextend.i64 v63 +;; @0025 v69 = icmp.i64 ugt v9, v64 +;; @0025 trapnz v69, user17 +;; @0025 v78 = load.i32 notrap aligned v0+56 +;; @0025 v79 = uextend.i64 v78 +;; @0025 v84 = icmp.i64 ugt v11, v79 +;; @0025 trapnz v84, heap_oob +;; @0025 v86 = load.i64 notrap aligned v0+48 +;; @0025 v93 = load.i64 notrap aligned v163+40 +;; v111 = iconst.i64 20 +;; @0025 v73 = iadd v60, v111 ; v111 = 20 +;; @0025 v95 = uadd_overflow_trap v73, v9, user2 +;; @0025 v94 = iadd v164, v93 +;; @0025 v96 = icmp ugt v95, v94 +;; @0025 trapnz v96, user2 +;; @0025 v88 = iadd v86, v8 +;; @0025 call fn1(v0, v73, v88, v9), stack_map=[i32 @ ss0+0] +;; v98 = load.i32 notrap v121 ;; @0029 jump block1 ;; ;; block1: -;; @0029 return v96 +;; @0029 return v98 ;; } diff --git a/tests/disas/gc/array-new-default-anyref.wat b/tests/disas/gc/array-new-default-anyref.wat index 82015ac7691f..c80b3f392509 100644 --- a/tests/disas/gc/array-new-default-anyref.wat +++ b/tests/disas/gc/array-new-default-anyref.wat @@ -10,7 +10,6 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { -;; ss0 = explicit_slot 4, align = 4 ;; region0 = 2 "vmctx" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly gv0+8 @@ -20,22 +19,20 @@ ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 ;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail -;; sig1 = (i64 vmctx, i64, i32, i64) tail ;; fn0 = colocated u805306368:24 sig0 -;; fn1 = colocated u805306368:2 sig1 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32): ;; @001f v5 = uextend.i64 v2 -;; v102 = iconst.i64 2 -;; v103 = ishl v5, v102 ; v102 = 2 -;; v100 = iconst.i64 32 -;; @001f v7 = ushr v103, v100 ; v100 = 32 +;; v98 = iconst.i64 2 +;; v99 = ishl v5, v98 ; v98 = 2 +;; v96 = iconst.i64 32 +;; @001f v7 = ushr v99, v96 ; v96 = 32 ;; @001f trapnz v7, user18 ;; @001f v4 = iconst.i32 20 -;; v109 = iconst.i32 2 -;; v110 = ishl v2, v109 ; v109 = 2 -;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20 +;; v105 = iconst.i32 2 +;; v106 = ishl v2, v105 ; v105 = 2 +;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20 ;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 ;; @001f v12 = load.i32 notrap aligned v11 ;; @001f v13 = load.i32 notrap aligned v11+4 @@ -51,22 +48,22 @@ ;; @001f brif v22, block2, block3 ;; ;; block2: -;; v118 = iconst.i32 15 -;; v119 = iadd.i32 v9, v118 ; v118 = 15 -;; v122 = iconst.i32 -16 -;; v123 = band v119, v122 ; v122 = -16 -;; v125 = iadd.i32 v12, v123 -;; @001f store notrap aligned region0 v125, v11 -;; v141 = iconst.i32 -1476394994 -;; v142 = load.i64 notrap aligned readonly can_move v0+8 -;; v143 = load.i64 notrap aligned readonly can_move v142+32 -;; @001f v36 = iadd v143, v19 -;; @001f store notrap aligned v141, v36 ; v141 = -1476394994 -;; v144 = load.i64 notrap aligned readonly can_move v0+40 -;; v145 = load.i32 notrap aligned readonly can_move v144 -;; @001f store notrap aligned v145, v36+4 -;; v146 = band.i64 v17, v16 ; v16 = -16 -;; @001f istore32 notrap aligned v146, v36+8 +;; v114 = iconst.i32 15 +;; v115 = iadd.i32 v9, v114 ; v114 = 15 +;; v118 = iconst.i32 -16 +;; v119 = band v115, v118 ; v118 = -16 +;; v121 = iadd.i32 v12, v119 +;; @001f store notrap aligned region0 v121, v11 +;; v137 = iconst.i32 -1476394994 +;; v138 = load.i64 notrap aligned readonly can_move v0+8 +;; v139 = load.i64 notrap aligned readonly can_move v138+32 +;; @001f v36 = iadd v139, v19 +;; @001f store notrap aligned v137, v36 ; v137 = -1476394994 +;; v140 = load.i64 notrap aligned readonly can_move v0+40 +;; v141 = load.i32 notrap aligned readonly can_move v140 +;; @001f store notrap aligned v141, v36+4 +;; v142 = band.i64 v17, v16 ; v16 = -16 +;; @001f istore32 notrap aligned v142, v36+8 ;; @001f jump block4(v12, v36) ;; ;; block3 cold: @@ -75,41 +72,51 @@ ;; @001f v27 = load.i32 notrap aligned readonly can_move v26 ;; @001f v28 = iconst.i32 16 ;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 -;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8 -;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32 +;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8 +;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32 ;; @001f v31 = uextend.i64 v29 ;; @001f v32 = iadd v30, v31 ;; @001f jump block4(v29, v32) ;; ;; block4(v41: i32, v42: i64): -;; v95 = stack_addr.i64 ss0 -;; store notrap v41, v95 -;; v94 = iconst.i64 16 -;; @001f v43 = iadd v42, v94 ; v94 = 16 +;; v91 = iconst.i64 16 +;; @001f v43 = iadd v42, v91 ; v91 = 16 ;; @001f store.i32 user2 v2, v43 -;; v77 = load.i32 notrap v95 -;; @001f trapz v77, user16 -;; v147 = load.i64 notrap aligned readonly can_move v0+8 -;; v148 = load.i64 notrap aligned readonly can_move v147+32 -;; @001f v46 = uextend.i64 v77 -;; @001f v48 = iadd v148, v46 -;; @001f v50 = iadd v48, v94 ; v94 = 16 +;; @001f trapz v41, user16 +;; v143 = load.i64 notrap aligned readonly can_move v0+8 +;; v144 = load.i64 notrap aligned readonly can_move v143+32 +;; @001f v46 = uextend.i64 v41 +;; @001f v48 = iadd v144, v46 +;; @001f v50 = iadd v48, v91 ; v91 = 16 ;; @001f v51 = load.i32 user2 readonly v50 ;; @001f v52 = uextend.i64 v51 ;; @001f v57 = icmp.i64 ugt v5, v52 ;; @001f trapnz v57, user17 -;; @001f v68 = load.i64 notrap aligned v147+40 +;; @001f v68 = load.i64 notrap aligned v143+40 ;; v85 = iconst.i64 20 ;; @001f v61 = iadd v48, v85 ; v85 = 20 -;; @001f v70 = uadd_overflow_trap v61, v103, user2 -;; @001f v69 = iadd v148, v68 +;; @001f v70 = uadd_overflow_trap v61, v99, user2 +;; @001f v69 = iadd v144, v68 ;; @001f v71 = icmp ugt v70, v69 ;; @001f trapnz v71, user2 +;; v123 = iconst.i64 0 +;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0 ;; @001f v44 = iconst.i32 0 -;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0 -;; v74 = load.i32 notrap v95 -;; @0022 jump block1 +;; v97 = iconst.i64 4 +;; @001f v72 = iadd v61, v99 +;; @001f brif v73, block6, block5(v61) ;; -;; block1: -;; @0022 return v74 +;; block5(v74: i64): +;; v145 = iconst.i32 0 +;; @001f store user2 little v145, v74 ; v145 = 0 +;; v146 = iconst.i64 4 +;; v147 = iadd v74, v146 ; v146 = 4 +;; @001f v76 = icmp eq v147, v72 +;; @001f brif v76, block6, block5(v147) +;; +;; block6: +;; @0022 jump block1(v41) +;; +;; block1(v3: i32): +;; @0022 return v3 ;; } diff --git a/tests/disas/gc/array-new-default-exnref.wat b/tests/disas/gc/array-new-default-exnref.wat index 910d21579e02..10e6a6d2a3bd 100644 --- a/tests/disas/gc/array-new-default-exnref.wat +++ b/tests/disas/gc/array-new-default-exnref.wat @@ -10,7 +10,6 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { -;; ss0 = explicit_slot 4, align = 4 ;; region0 = 2 "vmctx" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly gv0+8 @@ -20,22 +19,20 @@ ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 ;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail -;; sig1 = (i64 vmctx, i64, i32, i64) tail ;; fn0 = colocated u805306368:24 sig0 -;; fn1 = colocated u805306368:2 sig1 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32): ;; @001f v5 = uextend.i64 v2 -;; v102 = iconst.i64 2 -;; v103 = ishl v5, v102 ; v102 = 2 -;; v100 = iconst.i64 32 -;; @001f v7 = ushr v103, v100 ; v100 = 32 +;; v98 = iconst.i64 2 +;; v99 = ishl v5, v98 ; v98 = 2 +;; v96 = iconst.i64 32 +;; @001f v7 = ushr v99, v96 ; v96 = 32 ;; @001f trapnz v7, user18 ;; @001f v4 = iconst.i32 20 -;; v109 = iconst.i32 2 -;; v110 = ishl v2, v109 ; v109 = 2 -;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20 +;; v105 = iconst.i32 2 +;; v106 = ishl v2, v105 ; v105 = 2 +;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20 ;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 ;; @001f v12 = load.i32 notrap aligned v11 ;; @001f v13 = load.i32 notrap aligned v11+4 @@ -51,22 +48,22 @@ ;; @001f brif v22, block2, block3 ;; ;; block2: -;; v118 = iconst.i32 15 -;; v119 = iadd.i32 v9, v118 ; v118 = 15 -;; v122 = iconst.i32 -16 -;; v123 = band v119, v122 ; v122 = -16 -;; v125 = iadd.i32 v12, v123 -;; @001f store notrap aligned region0 v125, v11 -;; v141 = iconst.i32 -1476394994 -;; v142 = load.i64 notrap aligned readonly can_move v0+8 -;; v143 = load.i64 notrap aligned readonly can_move v142+32 -;; @001f v36 = iadd v143, v19 -;; @001f store notrap aligned v141, v36 ; v141 = -1476394994 -;; v144 = load.i64 notrap aligned readonly can_move v0+40 -;; v145 = load.i32 notrap aligned readonly can_move v144 -;; @001f store notrap aligned v145, v36+4 -;; v146 = band.i64 v17, v16 ; v16 = -16 -;; @001f istore32 notrap aligned v146, v36+8 +;; v114 = iconst.i32 15 +;; v115 = iadd.i32 v9, v114 ; v114 = 15 +;; v118 = iconst.i32 -16 +;; v119 = band v115, v118 ; v118 = -16 +;; v121 = iadd.i32 v12, v119 +;; @001f store notrap aligned region0 v121, v11 +;; v137 = iconst.i32 -1476394994 +;; v138 = load.i64 notrap aligned readonly can_move v0+8 +;; v139 = load.i64 notrap aligned readonly can_move v138+32 +;; @001f v36 = iadd v139, v19 +;; @001f store notrap aligned v137, v36 ; v137 = -1476394994 +;; v140 = load.i64 notrap aligned readonly can_move v0+40 +;; v141 = load.i32 notrap aligned readonly can_move v140 +;; @001f store notrap aligned v141, v36+4 +;; v142 = band.i64 v17, v16 ; v16 = -16 +;; @001f istore32 notrap aligned v142, v36+8 ;; @001f jump block4(v12, v36) ;; ;; block3 cold: @@ -75,41 +72,51 @@ ;; @001f v27 = load.i32 notrap aligned readonly can_move v26 ;; @001f v28 = iconst.i32 16 ;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 -;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8 -;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32 +;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8 +;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32 ;; @001f v31 = uextend.i64 v29 ;; @001f v32 = iadd v30, v31 ;; @001f jump block4(v29, v32) ;; ;; block4(v41: i32, v42: i64): -;; v95 = stack_addr.i64 ss0 -;; store notrap v41, v95 -;; v94 = iconst.i64 16 -;; @001f v43 = iadd v42, v94 ; v94 = 16 +;; v91 = iconst.i64 16 +;; @001f v43 = iadd v42, v91 ; v91 = 16 ;; @001f store.i32 user2 v2, v43 -;; v77 = load.i32 notrap v95 -;; @001f trapz v77, user16 -;; v147 = load.i64 notrap aligned readonly can_move v0+8 -;; v148 = load.i64 notrap aligned readonly can_move v147+32 -;; @001f v46 = uextend.i64 v77 -;; @001f v48 = iadd v148, v46 -;; @001f v50 = iadd v48, v94 ; v94 = 16 +;; @001f trapz v41, user16 +;; v143 = load.i64 notrap aligned readonly can_move v0+8 +;; v144 = load.i64 notrap aligned readonly can_move v143+32 +;; @001f v46 = uextend.i64 v41 +;; @001f v48 = iadd v144, v46 +;; @001f v50 = iadd v48, v91 ; v91 = 16 ;; @001f v51 = load.i32 user2 readonly v50 ;; @001f v52 = uextend.i64 v51 ;; @001f v57 = icmp.i64 ugt v5, v52 ;; @001f trapnz v57, user17 -;; @001f v68 = load.i64 notrap aligned v147+40 +;; @001f v68 = load.i64 notrap aligned v143+40 ;; v85 = iconst.i64 20 ;; @001f v61 = iadd v48, v85 ; v85 = 20 -;; @001f v70 = uadd_overflow_trap v61, v103, user2 -;; @001f v69 = iadd v148, v68 +;; @001f v70 = uadd_overflow_trap v61, v99, user2 +;; @001f v69 = iadd v144, v68 ;; @001f v71 = icmp ugt v70, v69 ;; @001f trapnz v71, user2 +;; v123 = iconst.i64 0 +;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0 ;; @001f v44 = iconst.i32 0 -;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0 -;; v74 = load.i32 notrap v95 -;; @0022 jump block1 +;; v97 = iconst.i64 4 +;; @001f v72 = iadd v61, v99 +;; @001f brif v73, block6, block5(v61) ;; -;; block1: -;; @0022 return v74 +;; block5(v74: i64): +;; v145 = iconst.i32 0 +;; @001f store user2 little v145, v74 ; v145 = 0 +;; v146 = iconst.i64 4 +;; v147 = iadd v74, v146 ; v146 = 4 +;; @001f v76 = icmp eq v147, v72 +;; @001f brif v76, block6, block5(v147) +;; +;; block6: +;; @0022 jump block1(v41) +;; +;; block1(v3: i32): +;; @0022 return v3 ;; } diff --git a/tests/disas/gc/array-new-default-externref.wat b/tests/disas/gc/array-new-default-externref.wat index 4a0771b172c0..bb5b2a4e564d 100644 --- a/tests/disas/gc/array-new-default-externref.wat +++ b/tests/disas/gc/array-new-default-externref.wat @@ -10,7 +10,6 @@ ) ) ;; function u0:0(i64 vmctx, i64, i32) -> i32 tail { -;; ss0 = explicit_slot 4, align = 4 ;; region0 = 2 "vmctx" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly gv0+8 @@ -20,22 +19,20 @@ ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 ;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail -;; sig1 = (i64 vmctx, i64, i32, i64) tail ;; fn0 = colocated u805306368:24 sig0 -;; fn1 = colocated u805306368:2 sig1 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32): ;; @001f v5 = uextend.i64 v2 -;; v102 = iconst.i64 2 -;; v103 = ishl v5, v102 ; v102 = 2 -;; v100 = iconst.i64 32 -;; @001f v7 = ushr v103, v100 ; v100 = 32 +;; v98 = iconst.i64 2 +;; v99 = ishl v5, v98 ; v98 = 2 +;; v96 = iconst.i64 32 +;; @001f v7 = ushr v99, v96 ; v96 = 32 ;; @001f trapnz v7, user18 ;; @001f v4 = iconst.i32 20 -;; v109 = iconst.i32 2 -;; v110 = ishl v2, v109 ; v109 = 2 -;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20 +;; v105 = iconst.i32 2 +;; v106 = ishl v2, v105 ; v105 = 2 +;; @001f v9 = uadd_overflow_trap v4, v106, user18 ; v4 = 20 ;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 ;; @001f v12 = load.i32 notrap aligned v11 ;; @001f v13 = load.i32 notrap aligned v11+4 @@ -51,22 +48,22 @@ ;; @001f brif v22, block2, block3 ;; ;; block2: -;; v118 = iconst.i32 15 -;; v119 = iadd.i32 v9, v118 ; v118 = 15 -;; v122 = iconst.i32 -16 -;; v123 = band v119, v122 ; v122 = -16 -;; v125 = iadd.i32 v12, v123 -;; @001f store notrap aligned region0 v125, v11 -;; v141 = iconst.i32 -1476394994 -;; v142 = load.i64 notrap aligned readonly can_move v0+8 -;; v143 = load.i64 notrap aligned readonly can_move v142+32 -;; @001f v36 = iadd v143, v19 -;; @001f store notrap aligned v141, v36 ; v141 = -1476394994 -;; v144 = load.i64 notrap aligned readonly can_move v0+40 -;; v145 = load.i32 notrap aligned readonly can_move v144 -;; @001f store notrap aligned v145, v36+4 -;; v146 = band.i64 v17, v16 ; v16 = -16 -;; @001f istore32 notrap aligned v146, v36+8 +;; v114 = iconst.i32 15 +;; v115 = iadd.i32 v9, v114 ; v114 = 15 +;; v118 = iconst.i32 -16 +;; v119 = band v115, v118 ; v118 = -16 +;; v121 = iadd.i32 v12, v119 +;; @001f store notrap aligned region0 v121, v11 +;; v137 = iconst.i32 -1476394994 +;; v138 = load.i64 notrap aligned readonly can_move v0+8 +;; v139 = load.i64 notrap aligned readonly can_move v138+32 +;; @001f v36 = iadd v139, v19 +;; @001f store notrap aligned v137, v36 ; v137 = -1476394994 +;; v140 = load.i64 notrap aligned readonly can_move v0+40 +;; v141 = load.i32 notrap aligned readonly can_move v140 +;; @001f store notrap aligned v141, v36+4 +;; v142 = band.i64 v17, v16 ; v16 = -16 +;; @001f istore32 notrap aligned v142, v36+8 ;; @001f jump block4(v12, v36) ;; ;; block3 cold: @@ -75,41 +72,51 @@ ;; @001f v27 = load.i32 notrap aligned readonly can_move v26 ;; @001f v28 = iconst.i32 16 ;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476394994, v28 = 16 -;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8 -;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32 +;; @001f v92 = load.i64 notrap aligned readonly can_move v0+8 +;; @001f v30 = load.i64 notrap aligned readonly can_move v92+32 ;; @001f v31 = uextend.i64 v29 ;; @001f v32 = iadd v30, v31 ;; @001f jump block4(v29, v32) ;; ;; block4(v41: i32, v42: i64): -;; v95 = stack_addr.i64 ss0 -;; store notrap v41, v95 -;; v94 = iconst.i64 16 -;; @001f v43 = iadd v42, v94 ; v94 = 16 +;; v91 = iconst.i64 16 +;; @001f v43 = iadd v42, v91 ; v91 = 16 ;; @001f store.i32 user2 v2, v43 -;; v77 = load.i32 notrap v95 -;; @001f trapz v77, user16 -;; v147 = load.i64 notrap aligned readonly can_move v0+8 -;; v148 = load.i64 notrap aligned readonly can_move v147+32 -;; @001f v46 = uextend.i64 v77 -;; @001f v48 = iadd v148, v46 -;; @001f v50 = iadd v48, v94 ; v94 = 16 +;; @001f trapz v41, user16 +;; v143 = load.i64 notrap aligned readonly can_move v0+8 +;; v144 = load.i64 notrap aligned readonly can_move v143+32 +;; @001f v46 = uextend.i64 v41 +;; @001f v48 = iadd v144, v46 +;; @001f v50 = iadd v48, v91 ; v91 = 16 ;; @001f v51 = load.i32 user2 readonly v50 ;; @001f v52 = uextend.i64 v51 ;; @001f v57 = icmp.i64 ugt v5, v52 ;; @001f trapnz v57, user17 -;; @001f v68 = load.i64 notrap aligned v147+40 +;; @001f v68 = load.i64 notrap aligned v143+40 ;; v85 = iconst.i64 20 ;; @001f v61 = iadd v48, v85 ; v85 = 20 -;; @001f v70 = uadd_overflow_trap v61, v103, user2 -;; @001f v69 = iadd v148, v68 +;; @001f v70 = uadd_overflow_trap v61, v99, user2 +;; @001f v69 = iadd v144, v68 ;; @001f v71 = icmp ugt v70, v69 ;; @001f trapnz v71, user2 +;; v123 = iconst.i64 0 +;; @001f v73 = icmp.i64 eq v5, v123 ; v123 = 0 ;; @001f v44 = iconst.i32 0 -;; @001f call fn1(v0, v61, v44, v103), stack_map=[i32 @ ss0+0] ; v44 = 0 -;; v74 = load.i32 notrap v95 -;; @0022 jump block1 +;; v97 = iconst.i64 4 +;; @001f v72 = iadd v61, v99 +;; @001f brif v73, block6, block5(v61) ;; -;; block1: -;; @0022 return v74 +;; block5(v74: i64): +;; v145 = iconst.i32 0 +;; @001f store user2 little v145, v74 ; v145 = 0 +;; v146 = iconst.i64 4 +;; v147 = iadd v74, v146 ; v146 = 4 +;; @001f v76 = icmp eq v147, v72 +;; @001f brif v76, block6, block5(v147) +;; +;; block6: +;; @0022 jump block1(v41) +;; +;; block1(v3: i32): +;; @0022 return v3 ;; } diff --git a/tests/disas/gc/array-new-default-funcref.wat b/tests/disas/gc/array-new-default-funcref.wat index baa936d1ddfd..45901103e3ca 100644 --- a/tests/disas/gc/array-new-default-funcref.wat +++ b/tests/disas/gc/array-new-default-funcref.wat @@ -20,22 +20,22 @@ ;; gv5 = load.i64 notrap aligned readonly can_move gv4+32 ;; gv6 = load.i64 notrap aligned gv4+40 ;; sig0 = (i64 vmctx, i32, i32, i32, i32) -> i32 tail -;; sig1 = (i64 vmctx, i64, i32, i64) tail +;; sig1 = (i64 vmctx, i64) -> i64 tail ;; fn0 = colocated u805306368:24 sig0 -;; fn1 = colocated u805306368:2 sig1 +;; fn1 = colocated u805306368:25 sig1 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32): ;; @001f v5 = uextend.i64 v2 -;; v102 = iconst.i64 2 -;; v103 = ishl v5, v102 ; v102 = 2 -;; v100 = iconst.i64 32 -;; @001f v7 = ushr v103, v100 ; v100 = 32 +;; v110 = iconst.i64 2 +;; v111 = ishl v5, v110 ; v110 = 2 +;; v108 = iconst.i64 32 +;; @001f v7 = ushr v111, v108 ; v108 = 32 ;; @001f trapnz v7, user18 ;; @001f v4 = iconst.i32 20 -;; v109 = iconst.i32 2 -;; v110 = ishl v2, v109 ; v109 = 2 -;; @001f v9 = uadd_overflow_trap v4, v110, user18 ; v4 = 20 +;; v117 = iconst.i32 2 +;; v118 = ishl v2, v117 ; v117 = 2 +;; @001f v9 = uadd_overflow_trap v4, v118, user18 ; v4 = 20 ;; @001f v11 = load.i64 notrap aligned readonly can_move v0+32 ;; @001f v12 = load.i32 notrap aligned v11 ;; @001f v13 = load.i32 notrap aligned v11+4 @@ -51,22 +51,22 @@ ;; @001f brif v22, block2, block3 ;; ;; block2: -;; v118 = iconst.i32 15 -;; v119 = iadd.i32 v9, v118 ; v118 = 15 -;; v122 = iconst.i32 -16 -;; v123 = band v119, v122 ; v122 = -16 -;; v125 = iadd.i32 v12, v123 -;; @001f store notrap aligned region0 v125, v11 -;; v140 = iconst.i32 -1476395002 -;; v141 = load.i64 notrap aligned readonly can_move v0+8 -;; v142 = load.i64 notrap aligned readonly can_move v141+32 -;; @001f v36 = iadd v142, v19 -;; @001f store notrap aligned v140, v36 ; v140 = -1476395002 -;; v143 = load.i64 notrap aligned readonly can_move v0+40 -;; v144 = load.i32 notrap aligned readonly can_move v143 -;; @001f store notrap aligned v144, v36+4 -;; v145 = band.i64 v17, v16 ; v16 = -16 -;; @001f istore32 notrap aligned v145, v36+8 +;; v126 = iconst.i32 15 +;; v127 = iadd.i32 v9, v126 ; v126 = 15 +;; v130 = iconst.i32 -16 +;; v131 = band v127, v130 ; v130 = -16 +;; v133 = iadd.i32 v12, v131 +;; @001f store notrap aligned region0 v133, v11 +;; v148 = iconst.i32 -1476395002 +;; v149 = load.i64 notrap aligned readonly can_move v0+8 +;; v150 = load.i64 notrap aligned readonly can_move v149+32 +;; @001f v36 = iadd v150, v19 +;; @001f store notrap aligned v148, v36 ; v148 = -1476395002 +;; v151 = load.i64 notrap aligned readonly can_move v0+40 +;; v152 = load.i32 notrap aligned readonly can_move v151 +;; @001f store notrap aligned v152, v36+4 +;; v153 = band.i64 v17, v16 ; v16 = -16 +;; @001f istore32 notrap aligned v153, v36+8 ;; @001f jump block4(v12, v36) ;; ;; block3 cold: @@ -75,41 +75,56 @@ ;; @001f v27 = load.i32 notrap aligned readonly can_move v26 ;; @001f v28 = iconst.i32 16 ;; @001f v29 = call fn0(v0, v24, v27, v9, v28) ; v24 = -1476395002, v28 = 16 -;; @001f v96 = load.i64 notrap aligned readonly can_move v0+8 -;; @001f v30 = load.i64 notrap aligned readonly can_move v96+32 +;; @001f v104 = load.i64 notrap aligned readonly can_move v0+8 +;; @001f v30 = load.i64 notrap aligned readonly can_move v104+32 ;; @001f v31 = uextend.i64 v29 ;; @001f v32 = iadd v30, v31 ;; @001f jump block4(v29, v32) ;; ;; block4(v41: i32, v42: i64): -;; v95 = stack_addr.i64 ss0 -;; store notrap v41, v95 -;; v94 = iconst.i64 16 -;; @001f v43 = iadd v42, v94 ; v94 = 16 +;; v103 = stack_addr.i64 ss0 +;; store notrap v41, v103 +;; v102 = iconst.i64 16 +;; @001f v43 = iadd v42, v102 ; v102 = 16 ;; @001f store.i32 user2 v2, v43 -;; v77 = load.i32 notrap v95 -;; @001f trapz v77, user16 -;; v146 = load.i64 notrap aligned readonly can_move v0+8 -;; v147 = load.i64 notrap aligned readonly can_move v146+32 -;; @001f v46 = uextend.i64 v77 -;; @001f v48 = iadd v147, v46 -;; @001f v50 = iadd v48, v94 ; v94 = 16 +;; v83 = load.i32 notrap v103 +;; @001f trapz v83, user16 +;; v154 = load.i64 notrap aligned readonly can_move v0+8 +;; v155 = load.i64 notrap aligned readonly can_move v154+32 +;; @001f v46 = uextend.i64 v83 +;; @001f v48 = iadd v155, v46 +;; @001f v50 = iadd v48, v102 ; v102 = 16 ;; @001f v51 = load.i32 user2 readonly v50 ;; @001f v52 = uextend.i64 v51 ;; @001f v57 = icmp.i64 ugt v5, v52 ;; @001f trapnz v57, user17 -;; @001f v68 = load.i64 notrap aligned v146+40 -;; v85 = iconst.i64 20 -;; @001f v61 = iadd v48, v85 ; v85 = 20 -;; @001f v70 = uadd_overflow_trap v61, v103, user2 -;; @001f v69 = iadd v147, v68 +;; @001f v68 = load.i64 notrap aligned v154+40 +;; v93 = iconst.i64 20 +;; @001f v61 = iadd v48, v93 ; v93 = 20 +;; @001f v70 = uadd_overflow_trap v61, v111, user2 +;; @001f v69 = iadd v155, v68 ;; @001f v71 = icmp ugt v70, v69 ;; @001f trapnz v71, user2 -;; @001f v45 = iconst.i32 0 -;; @001f call fn1(v0, v61, v45, v103), stack_map=[i32 @ ss0+0] ; v45 = 0 -;; v74 = load.i32 notrap v95 +;; @001f v44 = iconst.i64 0 +;; @001f v73 = icmp.i64 eq v5, v44 ; v44 = 0 +;; v109 = iconst.i64 4 +;; @001f v72 = iadd v61, v111 +;; @001f brif v73, block6, block5(v61) +;; +;; block5(v74: i64): +;; v156 = iconst.i64 0 +;; @001f v76 = call fn1(v0, v156), stack_map=[i32 @ ss0+0] ; v156 = 0 +;; @001f v77 = ireduce.i32 v76 +;; @001f store user2 little v77, v74 +;; v157 = iconst.i64 4 +;; v158 = iadd v74, v157 ; v157 = 4 +;; @001f v79 = icmp eq v158, v72 +;; @001f brif v79, block6, block5(v158) +;; +;; block6: +;; v80 = load.i32 notrap v103 ;; @0022 jump block1 ;; ;; block1: -;; @0022 return v74 +;; @0022 return v80 ;; } diff --git a/tests/disas/passive-data.wat b/tests/disas/passive-data.wat index 0fbe3b45fcbe..0d8605e9c419 100644 --- a/tests/disas/passive-data.wat +++ b/tests/disas/passive-data.wat @@ -28,29 +28,30 @@ ;; @003d v6 = load.i64 notrap aligned v0+64 ;; @003d v7 = uextend.i64 v2 ;; @003d v8 = uextend.i64 v4 -;; v32 = iconst.i64 1 -;; @003d v9 = imul v8, v32 ; v32 = 1 +;; v33 = iconst.i64 1 +;; @003d v9 = imul v8, v33 ; v33 = 1 ;; @003d v10 = iadd v7, v9 ;; @003d v11 = icmp ugt v10, v6 ;; @003d trapnz v11, heap_oob ;; @003d v12 = load.i64 notrap aligned readonly can_move v0+56 ;; @003d v13 = uextend.i64 v2 -;; v30 = iconst.i64 1 -;; @003d v14 = imul v13, v30 ; v30 = 1 +;; v31 = iconst.i64 1 +;; @003d v14 = imul v13, v31 ; v31 = 1 ;; @003d v15 = iadd v12, v14 -;; @003d v17 = uload32 notrap aligned v0+152 -;; @003d v18 = uextend.i64 v3 -;; @003d v19 = uextend.i64 v4 -;; v29 = iconst.i64 1 -;; @003d v20 = imul v19, v29 ; v29 = 1 -;; @003d v21 = iadd v18, v20 -;; @003d v22 = icmp ugt v21, v17 -;; @003d trapnz v22, heap_oob -;; @003d v24 = load.i64 notrap aligned v0+144 -;; @003d v25 = uextend.i64 v3 -;; @003d v26 = iadd v24, v25 -;; @003d v27 = uextend.i64 v4 -;; @003d call fn0(v0, v15, v26, v27) +;; @003d v17 = load.i32 notrap aligned v0+152 +;; @003d v18 = uextend.i64 v17 +;; @003d v19 = uextend.i64 v3 +;; @003d v20 = uextend.i64 v4 +;; v30 = iconst.i64 1 +;; @003d v21 = imul v20, v30 ; v30 = 1 +;; @003d v22 = iadd v19, v21 +;; @003d v23 = icmp ugt v22, v18 +;; @003d trapnz v23, heap_oob +;; @003d v25 = load.i64 notrap aligned v0+144 +;; @003d v26 = uextend.i64 v3 +;; @003d v27 = iadd v25, v26 +;; @003d v28 = uextend.i64 v4 +;; @003d call fn0(v0, v15, v27, v28) ;; @0041 jump block1 ;; ;; block1: diff --git a/tests/disas/startup-data-active.wat b/tests/disas/startup-data-active.wat new file mode 100644 index 000000000000..95e2e5b187b0 --- /dev/null +++ b/tests/disas/startup-data-active.wat @@ -0,0 +1,62 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' +;;! flags = '-Wgc -Wfunction-references' + +(module + (memory 1) + + (data (i32.const 1) "hi") +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned gv0+64 +;; gv2 = load.i64 notrap aligned readonly can_move gv0+56 +;; sig0 = (i64 vmctx, i64, i64, i64) tail +;; fn0 = colocated u805306368:1 sig0 +;; +;; block0(v0: i64, v1: i64): +;; v3 = load.i64 notrap aligned v0+112 +;; v4 = iconst.i64 0 +;; v5 = icmp eq v3, v4 ; v4 = 0 +;; brif v5, block2, block1 +;; +;; block1: +;; v8 = load.i32 notrap aligned v0+120 +;; v11 = load.i64 notrap aligned v0+64 +;; v13 = uextend.i64 v8 +;; v16 = icmp ugt v13, v11 +;; trapnz v16, heap_oob +;; v17 = load.i64 notrap aligned readonly can_move v0+56 +;; call fn0(v0, v17, v3, v13) +;; jump block2 +;; +;; block2: +;; return +;; } diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat new file mode 100644 index 000000000000..10e6af2848d3 --- /dev/null +++ b/tests/disas/startup-elem-active.wat @@ -0,0 +1,77 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' +;;! flags = '-Wgc -Wfunction-references' + +(module + (table 10 anyref) + + (elem (i32.const 1) (ref i31) + (item (ref.i31 (i32.const 10))) + (item (ref.i31 (i32.const 11))) + (item (ref.i31 (i32.const 12))) + ) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; region0 = 1 "table" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned gv0+48 +;; gv2 = load.i64 notrap aligned gv0+56 +;; +;; block0(v0: i64, v1: i64): +;; v4 = load.i64 notrap aligned v0+56 +;; v5 = ireduce.i32 v4 +;; v6 = uextend.i64 v5 +;; v86 = iconst.i64 4 +;; v92 = icmp ult v6, v86 ; v86 = 4 +;; trapnz v92, user6 +;; v12 = load.i64 notrap aligned v0+48 +;; v103 = iconst.i32 21 +;; v2 = iconst.i32 1 +;; v114 = icmp ule v5, v2 ; v2 = 1 +;; v79 = iconst.i64 0 +;; v15 = iadd v12, v86 ; v86 = 4 +;; v28 = select_spectre_guard v114, v79, v15 ; v79 = 0 +;; store user6 aligned region0 v103, v28 ; v103 = 21 +;; v117 = iconst.i32 23 +;; v123 = iconst.i32 2 +;; v129 = icmp ule v5, v123 ; v123 = 2 +;; v131 = iconst.i64 8 +;; v39 = iadd v12, v131 ; v131 = 8 +;; v41 = select_spectre_guard v129, v79, v39 ; v79 = 0 +;; store user6 aligned region0 v117, v41 ; v117 = 23 +;; v133 = iconst.i32 25 +;; v3 = iconst.i32 3 +;; v144 = icmp ule v5, v3 ; v3 = 3 +;; v146 = iconst.i64 12 +;; v52 = iadd v12, v146 ; v146 = 12 +;; v54 = select_spectre_guard v144, v79, v52 ; v79 = 0 +;; store user6 aligned region0 v133, v54 ; v133 = 25 +;; return +;; } diff --git a/tests/disas/startup-global.wat b/tests/disas/startup-global.wat new file mode 100644 index 000000000000..49da0905dfc8 --- /dev/null +++ b/tests/disas/startup-global.wat @@ -0,0 +1,42 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' + +(module + (global i64 i64.const 0 i64.const 0 i64.add) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; region0 = 1 "table" +;; gv0 = vmctx +;; +;; block0(v0: i64, v1: i64): +;; v2 = iconst.i64 0 +;; store notrap aligned region0 v2, v0+48 ; v2 = 0 +;; return +;; } diff --git a/tests/disas/startup-passive-segment.wat b/tests/disas/startup-passive-segment.wat new file mode 100644 index 000000000000..eaec68a40327 --- /dev/null +++ b/tests/disas/startup-passive-segment.wat @@ -0,0 +1,50 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' +;;! flags = '-Wgc' + +(module + (elem (ref i31) (item (ref.i31 (i32.const 0))) (item (ref.i31 (i32.const 1)))) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; gv0 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; fn0 = colocated u805306368:4 sig0 +;; +;; block0(v0: i64, v1: i64): +;; v3 = iconst.i32 0 +;; v4 = call fn0(v0, v3) ; v3 = 0 +;; v18 = iconst.i32 1 +;; store user2 little v18, v4 ; v18 = 1 +;; v25 = iconst.i32 3 +;; v13 = iconst.i64 16 +;; v12 = iadd v4, v13 ; v13 = 16 +;; store user2 little v25, v12 ; v25 = 3 +;; return +;; } diff --git a/tests/disas/startup-start.wat b/tests/disas/startup-start.wat new file mode 100644 index 000000000000..aac06f24462c --- /dev/null +++ b/tests/disas/startup-start.wat @@ -0,0 +1,42 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' + +(module + (func $f) + (start $f) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u0:0 sig0 +;; +;; block0(v0: i64, v1: i64): +;; call fn0(v0, v0) +;; return +;; } diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat new file mode 100644 index 000000000000..657a42ecbb54 --- /dev/null +++ b/tests/disas/startup-table-initial-value.wat @@ -0,0 +1,65 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! filter = 'module_start' +;;! flags = '-Wgc -Wfunction-references' + +(module + (table 10 (ref i31) (ref.i31 (i32.const 0))) +) +;; function u2415919104:1(i64 vmctx, i64, i64, i64) -> i8 system_v { +;; sig0 = (i64 vmctx, i64) tail +;; fn0 = colocated u2415919104:0 sig0 +;; +;; block0(v0: i64, v1: i64, v2: i64, v3: i64): +;; jump block1 +;; +;; block1: +;; v4 = load.i64 notrap aligned v0+8 +;; v5 = get_frame_pointer.i64 +;; store notrap aligned v5, v4+72 +;; v6 = get_stack_pointer.i64 +;; store notrap aligned v6, v4+64 +;; v7 = get_exception_handler_address.i64 block1, 0 +;; store notrap aligned v7, v4+80 +;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ] +;; +;; block2: +;; v8 = iconst.i8 1 +;; return v8 ; v8 = 1 +;; +;; block3: +;; v9 = iconst.i8 0 +;; return v9 ; v9 = 0 +;; } +;; +;; function u2415919104:0(i64 vmctx, i64) tail { +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned gv0+48 +;; gv2 = load.i64 notrap aligned gv0+56 +;; +;; block0(v0: i64, v1: i64): +;; v7 = load.i64 notrap aligned v0+56 +;; v8 = ireduce.i32 v7 +;; v9 = uextend.i64 v8 +;; v41 = iconst.i64 10 +;; v53 = icmp ult v9, v41 ; v41 = 10 +;; trapnz v53, user6 +;; v15 = load.i64 notrap aligned v0+48 +;; v34 = iconst.i32 1 +;; v83 = iconst.i64 36 +;; v85 = iadd v15, v83 ; v83 = 36 +;; v29 = iconst.i64 4 +;; jump block1(v15) +;; +;; block1(v23: i64): +;; v88 = iconst.i32 1 +;; store notrap aligned v88, v23 ; v88 = 1 +;; v89 = iadd.i64 v15, v83 ; v83 = 36 +;; v90 = icmp eq v23, v89 +;; v91 = iconst.i64 4 +;; v92 = iadd v23, v91 ; v91 = 4 +;; brif v90, block2, block1(v92) +;; +;; block2: +;; return +;; } diff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.wat index ae2e8eaf4363..e9ccc89ad7c3 100644 --- a/tests/disas/winch/x64/table/init_copy_drop.wat +++ b/tests/disas/winch/x64/table/init_copy_drop.wat @@ -303,7 +303,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -388,7 +388,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -473,7 +473,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -558,7 +558,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -643,7 +643,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movq 8(%rsp), %rdx -;; callq 0x114d +;; callq 0x1178 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x28(%rsp), %r14 @@ -754,7 +754,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 0xc(%rsp), %edx -;; callq 0x114d +;; callq 0x1178 ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/misc_testsuite/gc/array-fill-null-funcref.wast b/tests/misc_testsuite/gc/array-fill-null-funcref.wast new file mode 100644 index 000000000000..2e74c3384d16 --- /dev/null +++ b/tests/misc_testsuite/gc/array-fill-null-funcref.wast @@ -0,0 +1,10 @@ +;;! gc = true + +(module + (type $a (array funcref)) + (func (export "hi") (result funcref) + (array.get $a (array.new_default $a (i32.const 10)) (i32.const 3)) + ) +) + +(assert_return (invoke "hi") (ref.null func)) diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index be5eaefc49e5..ffa64484d01c 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -1722,7 +1722,7 @@ where self.emit_bounds_check_and_compute_addr(&dst_heap, dst_raw_addr, dst.reg, len.reg)?; self.context.free_reg(dst); - let passive_data_index = match self.env.translation.passive_data_map[segment] { + let runtime_data_index = match self.env.translation.runtime_data_map[segment] { Some(i) => i, // Active data segments always have length zero, so this is only @@ -1747,7 +1747,7 @@ where let data_segment_length_offset = self .env .vmoffsets - .vmctx_passive_data_length(passive_data_index); + .vmctx_runtime_data_length(runtime_data_index); let tmp1 = self.context.any_gpr(self.masm)?; let tmp2 = self.context.any_gpr(self.masm)?; self.masm.load( @@ -1774,7 +1774,7 @@ where let data_segment_base_offset = self .env .vmoffsets - .vmctx_passive_data_base(passive_data_index); + .vmctx_runtime_data_base(runtime_data_index); self.masm.load( self.masm.address_at_vmctx(data_segment_base_offset)?, writable!(tmp1), @@ -1804,7 +1804,7 @@ where } pub fn emit_data_drop(&mut self, data_index: DataIndex) -> Result<()> { - let passive_data_index = match self.env.translation.passive_data_map[data_index] { + let runtime_data_index = match self.env.translation.runtime_data_map[data_index] { Some(idx) => idx, // Active data segments do nothing when dropped, so this is a noop. None => return Ok(()), @@ -1812,7 +1812,7 @@ where let data_segment_offset = self .env .vmoffsets - .vmctx_passive_data_length(passive_data_index); + .vmctx_runtime_data_length(runtime_data_index); let len_addr = self.masm.address_at_vmctx(data_segment_offset)?; self.masm.store(RegImm::i32(0), len_addr, OperandSize::S32) }