diff --git a/docs/design/mono/webcil.md b/docs/design/mono/webcil.md index afaf95689bd6e1..d5bcadda40c90d 100644 --- a/docs/design/mono/webcil.md +++ b/docs/design/mono/webcil.md @@ -80,10 +80,41 @@ If data segment 0 is at least 8 bytes in size, and the second 4 bytes has a non- little-endian unsigned 32-bit integer, then data segment 0 encodes two little-endian u32 values: `payloadSize` (first 4 bytes) and `tableSize` (second 4 bytes). In this case, `tableSize` shall be the number of table entries required for the WebAssembly -module to be loaded, and the module shall import a table, as well as `stackPointer`, `tableBase`, and -`imageBase` globals. There shall also be a `fillWebcilTable` function which will initialize the table -with appropriate values. The `getWebcilPayload` API shall be enhanced to fill in the `TableBase` field -of the `WebcilHeader`. +module to be loaded, and the module shall import a table, as well as `__stack_pointer`, `__table_base`, and +`__memory_base` globals. Two module shapes are permitted, distinguished by whether the payload and +table segments are passive or active. + +A **host-installed** module keeps both segments passive. It shall provide a `fillWebcilTable` +function which initializes the table, and its `getWebcilPayload` API shall copy the payload and fill +in the `TableBase` field of the `WebcilHeader`. Per-assembly component forwarding stubs use this +shape, because a stub may be parsed from its file rather than instantiated. + +A **self-installing** module emits the payload as an active data segment at `(global.get __memory_base)` +and the table as an active element segment at `(global.get __table_base)`, so the engine installs both +at instantiation. Such a module exports neither `getWebcilPayload` nor `fillWebcilTable` - calling +`memory.init` or `table.init` against an active segment traps, because an active segment is implicitly +dropped once applied. It shall instead export `patchWebcilHeader`, which fills in the `TableBase` field; +the host must call it after instantiation, since the runtime reads that field from the mapped image and +an unwritten field reads as 0, silently shifting every function index by `tableBase`. Composite and +single-assembly R2R images use this shape. + +A self-installing module leaves `__memory_base` and `__table_base` as imports, which a host may satisfy +in either of two ways, with different consequences: + +- **Supply them at instantiation**, as immutable `WebAssembly.Global` values. The segment offsets stay + `global.get` of an *imported* global, which is a valid constant expression, so the module needs no + further processing. The browser host does this. +- **Define and export them, then link the module into the host** with a merge tool. Merging internalizes + the globals, and `global.get` of a *defined* global is not a constant expression outside the GC + proposal - engines disagree here, so the merged module must have its offsets folded to `i32.const` + before it is portable. The offline WASI pipeline does this. + +Neither approach requires rewriting the segments themselves; only the second requires a fold, and that +fold is not free, because the pass that performs it also propagates globals into function bodies. + +A host that reserves the composite's table slice at link time can treat `__table_base` as a constant: +reserving the first N slots leaves the composite at base 1 regardless of its size. `__memory_base` is +the address of the host's payload region and has to be read out of the linked host. The memory of the WebcilPayload must also be allocated with 16 byte alignment. @@ -94,14 +125,15 @@ reachable. Function names shall instead be carried in the `name` custom section, engines, counts towards no limit, and may be stripped when size matters. ``` wat +;; Host-installed shape (passive segments). (module (data "\0f\00\00\00\01\00\00\00") ;; data segment 0: two little-endian u32 values (payloadSize, tableSize). This specifies a Webcil payload of size 15 bytes with 1 required table entry (data "webcil Payload\cc") ;; data segment 1: Webcil payload (import "webcil" "memory" (memory (;0;) 1)) - (import "webcil" "stackPointer" (global (;0;) (mut i32))) - (import "webcil" "imageBase" (global (;1;) i32)) - (import "webcil" "tableBase" (global (;2;) i32)) - (import "webcil" "table" (table (;0;) 1 funcref)) + (import "webcil" "__stack_pointer" (global (;0;) (mut i32))) + (import "webcil" "__memory_base" (global (;1;) i32)) + (import "webcil" "__table_base" (global (;2;) i32)) + (import "webcil" "__indirect_function_table" (table (;0;) 1 funcref)) (global (export "webcilVersion") i32 (i32.const 1)) (func (export "getWebcilSize") (param $destPtr i32) (result) local.get $destPtr @@ -134,6 +166,38 @@ engines, counts towards no limit, and may be stripped when size matters. (elem (;0;) func 3)) ``` +``` wat +;; Self-installing shape (active segments). The engine applies both segments at +;; instantiation, so only the header's TableBase field is left for the host to trigger. +(module + (data "\0f\00\00\00\01\00\00\00") ;; data segment 0: payloadSize, tableSize - stays passive, read from the file before instantiation + (data (global.get 1) "webcil Payload\cc") ;; data segment 1: Webcil payload, active at __memory_base + (import "webcil" "memory" (memory (;0;) 1)) + (import "webcil" "__stack_pointer" (global (;0;) (mut i32))) + (import "webcil" "__memory_base" (global (;1;) i32)) + (import "webcil" "__table_base" (global (;2;) i32)) + (import "webcil" "__indirect_function_table" (table (;0;) 1 funcref)) + (global (export "webcilVersion") i32 (i32.const 1)) + (func (export "getWebcilSize") (param $destPtr i32) (result) + local.get $destPtr + i32.const 0 + i32.const 4 + memory.init 0) + (func (export "patchWebcilHeader") (param $d i32) (param $n i32) (result) + local.get 1 + i32.const 32 + i32.ge_s + if + local.get 0 + global.get 2 + i32.store offset=28 + end + ) + (func (param $d i32) (result i32) + local.get 0) + (elem (;0;) (global.get 2) func 2)) ;; active at __table_base +``` + (**Rationale**: With this approach it is possible to identify without loading the webcil module exactly the allocations/table growth/globals which are needed to load the webcil module via instantiateStreaming without actually loading the module.) @@ -150,6 +214,29 @@ of the runtime's wasm code, reducing the volume of code needed in each webcil fi images into linear memory, as well as for allowing for efficient storage of 128 bit vector constants within the binary.) +##### WASI host composition + +WASI hosts use offline composition rather than instantiating R2R modules at runtime. +The composer in `src/mono/wasi/build/compose-r2r.py` unbundles the host component, creates a +shim exporting the image and table bases, merges the host, shim, and composite with +`wasm-merge`, folds the globals with `wasm-opt --simplify-globals`, and replaces the component's +core module. Both Binaryen steps preserve the `name` section with `-g`. + +The host exports its reserved buffer address, buffer capacity, and composite table base through +`wasi_r2r_image_base`, `wasi_r2r_image_cap`, and `wasi_r2r_table_base`. Before merging, the composer +checks that the active payload fits the buffer and that the composite's table entries end before +the host's own active element segment. These checks must happen before instantiation: a runtime +check cannot prevent an active segment from overwriting an undersized reservation. + +The shim's start function calls `patchWebcilHeader`; the active segments themselves install the +payload and function table. The host's external assembly probe serves `composite-r2r.wasm` from +the embedded buffer and extracts per-assembly forwarding stubs from `comp/.wasm`. +Placing a composite on disk without composing it into the host does not satisfy this contract. + +Publishing sizes the host reservations from the generated app/framework composite. Runtime tests +instead use the shared corerun's fixed reservations and compose only their test assemblies. +See [the WASI R2R workflow](../../workflow/building/coreclr/wasi-r2r.md) for usage and diagnostics. + ### Webcil payload The webcil payload contains the ECMA-335 metadata, IL and resources comprising a .NET assembly. diff --git a/docs/workflow/building/coreclr/README.md b/docs/workflow/building/coreclr/README.md index a9ddf55587034b..4861a09ac9a05b 100644 --- a/docs/workflow/building/coreclr/README.md +++ b/docs/workflow/building/coreclr/README.md @@ -96,6 +96,7 @@ Detailed instructions on how to do cross-compilation can be found in the cross-b For specialized platforms, detailed instructions are available in the following guides: - **WebAssembly:** [Building CoreCLR for WebAssembly](/docs/workflow/building/coreclr/wasm.md) - Experimental support for building, running, and debugging CoreCLR on WebAssembly. +- **WASI ReadyToRun:** [Publishing and testing composite R2R images](wasi-r2r.md) - Experimental CoreCLR-WASI composition workflow. ## Other Features diff --git a/docs/workflow/building/coreclr/wasi-r2r.md b/docs/workflow/building/coreclr/wasi-r2r.md new file mode 100644 index 00000000000000..8937a0dd467f4f --- /dev/null +++ b/docs/workflow/building/coreclr/wasi-r2r.md @@ -0,0 +1,78 @@ +# CoreCLR-WASI composite ReadyToRun + +This is an experimental in-tree publishing and runtime-test workflow. The shipping WASI SDK +does not yet select the CoreCLR app builder. + +WASI requires the compiled composite to be composed into the host component before execution. +Copying `composite-r2r.wasm` beside an unmodified host is not sufficient. The composer lives +alongside the WASI app-builder targets in `src/mono/wasi/build/compose-r2r.py`. +The [WebCIL design document](../../../design/mono/webcil.md#wasi-host-composition) describes +the image layout and host contract. + +## Prerequisites + +Build prerequisites are described in the [CoreCLR build guide](README.md). +Composition additionally requires Python 3.8+, `wasm-tools`, and Binaryen's `wasm-merge` and +`wasm-opt` on `PATH`. Running the result requires wasmtime with WebAssembly exception support. +CI provisions pinned tool versions through `eng/testing/wasi-r2r-provisioning.targets`. + +Framework-sized composites can require several GiB of memory during composition. Allow for the +host and composite working sets when sizing build containers. + +## Publishing + +Build the runtime, libraries, and packs, then publish an in-tree WASI project: + +```bash +./build.sh -s clr+libs+packs -os wasi -arch wasm -c Release +./dotnet.sh publish -c Release -p:TargetOS=wasi \ + -p:RuntimeFlavor=CoreCLR -p:PublishReadyToRun=true +``` + +The app builder enables composite R2R, compiles the app/framework closure, sizes the host's image +buffer and table reservation, links the host, and invokes the composer. It deploys the composed +host and the per-assembly stubs under the app bundle's `managed/` directory. +Non-composite R2R and `WasmSingleFileBundle` are not supported by this path. + +## Runtime tests + +The WASI runtime-test pipeline keeps separate interpreter and `R2R_CG2` jobs. The R2R job uses +Checked CoreCLR with Release libraries. For a local merged-runner example: + +```bash +./build.sh -s clr+libs+packs -os wasi -arch wasm -c Release -rc Checked -lc Release -hc Release +src/tests/build.sh -os wasi -arch wasm Checked -dir:JIT/CodeGenBringUpTests \ + -priority1 -crossgen2 /p:LibrariesConfiguration=Release /p:HostConfiguration=Release +export CORE_ROOT="$PWD/artifacts/tests/coreclr/wasi.wasm.Checked/Tests/Core_Root" +export __TestDotNetCmd="$PWD/.dotnet/dotnet" +RunCrossGen2=1 bash artifacts/tests/coreclr/wasi.wasm.Checked/JIT/CodeGenBringUpTests/JIT.CodeGenBringUpTests_ro/JIT.CodeGenBringUpTests_ro.sh +``` + +The wrapper compiles the runner and referenced test assemblies into a composite, then invokes +`CORE_ROOT/wasi-r2r/compose-r2r.py`. It launches `IL-CG2/wasi-r2r/corerun-composite.wasm` with +`APP_ASSEMBLIES=EXTERNAL` and `CORE_LIBRARIES=/IL-CG2/wasi-r2r` so the guest probes the private +`comp/` stubs. `TEST_READY_TO_RUN_MODE=1` enables R2R-specific test conditions. + +Without `RunCrossGen2`, the wrapper uses the original host and does not probe those stubs. +Composition failures fail the test rather than silently falling back to interpretation. +Helix receives the composition tools through a separate correlation payload. + +## Composer interface and diagnostics + +The build targets generate Crossgen2 response files; a hand-maintained response-file template +is not required. To inspect or compose existing outputs directly: + +```bash +python3 src/mono/wasi/build/compose-r2r.py --describe +COMP= CORERUN= OUTDIR= \ + python3 src/mono/wasi/build/compose-r2r.py +``` + +`--describe` reports `functionCount,payloadBytes`. Composition requires all three environment +variables and writes `corerun-composite.wasm` into `OUTDIR`. The host supplies the image and table +bases; the script rejects an undersized buffer or overlapping table reservation. + +A valid composed module and passing tests alone do not prove R2R was used. Enable the guest's +`DOTNET_ReadyToRunLogFile` and look for `Ready to Run initialized successfully` for the test +assemblies. This confirms image loading; proving a particular method executes compiled code +requires a breakpoint or trace in that method's wasm body. diff --git a/eng/pipelines/common/evaluate-default-paths.yml b/eng/pipelines/common/evaluate-default-paths.yml index dba1aaad202b60..3914622f79a180 100644 --- a/eng/pipelines/common/evaluate-default-paths.yml +++ b/eng/pipelines/common/evaluate-default-paths.yml @@ -295,11 +295,12 @@ jobs: - subset: wasm_coreclr_runtimetests combined: true include: + - eng/testing/wasi-r2r-provisioning.targets + - src/mono/wasi/build/compose-r2r.py - src/tests/* - src/coreclr/* - ${{ parameters._const_paths._wasm_src_native }} exclude: - - src/mono/* - ${{ parameters._const_paths._wasm_pipelines }} - ${{ parameters._const_paths._always_exclude }} - ${{ parameters._const_paths._perf_pipeline_specific_only }} diff --git a/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml index 6bdf98d58f9fa9..745b56ce898e41 100644 --- a/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml +++ b/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml @@ -112,7 +112,9 @@ steps: helixBuild: $(Build.BuildNumber) helixSource: $(_HelixSource) - ${{ if ne(parameters.readyToRun, true) }}: + ${{ if eq(parameters.readyToRun, true) }}: + helixType: 'test/functional/r2r/cli/' + ${{ else }}: helixType: 'test/functional/cli/' helixQueues: ${{ parameters.helixQueues }} diff --git a/eng/pipelines/common/templates/wasi-wasm-coreclr-runtime-tests.yml b/eng/pipelines/common/templates/wasi-wasm-coreclr-runtime-tests.yml index c9a025e8a7cb4d..2a18559d44fd4e 100644 --- a/eng/pipelines/common/templates/wasi-wasm-coreclr-runtime-tests.yml +++ b/eng/pipelines/common/templates/wasi-wasm-coreclr-runtime-tests.yml @@ -5,6 +5,7 @@ parameters: platforms: [] extraBuildArgs: '' useHelixMonitor: false + readyToRun: false jobs: @@ -22,7 +23,10 @@ jobs: parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - buildConfig: Release + ${{ if eq(parameters.readyToRun, true) }}: + buildConfig: Checked + ${{ else }}: + buildConfig: Release runtimeFlavor: coreclr platforms: ${{ parameters.platforms }} variables: @@ -42,8 +46,11 @@ jobs: jobParameters: testGroup: innerloop isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} - nameSuffix: CoreCLR_WASI_RuntimeTests - buildArgs: -s clr+libs+packs -c $(_BuildConfig) ${{ parameters.extraBuildArgs }} /p:TestAssemblies=false + ${{ if eq(parameters.readyToRun, true) }}: + nameSuffix: CoreCLR_WASI_RuntimeTests_R2R_CG2 + ${{ else }}: + nameSuffix: CoreCLR_WASI_RuntimeTests + buildArgs: -s clr+libs+packs -c $(_BuildConfig) -lc Release -hc Release ${{ parameters.extraBuildArgs }} /p:TestAssemblies=false timeoutInMinutes: 180 condition: >- or( @@ -55,8 +62,11 @@ jobs: creator: dotnet-bot testRunNamePrefixSuffix: CoreCLR_WASI_$(_BuildConfig) useHelixMonitor: ${{ parameters.useHelixMonitor }} + readyToRun: ${{ parameters.readyToRun }} + compositeBuildMode: ${{ parameters.readyToRun }} # Curated test-tree subset -- expand as gates stabilize. testBuildArgs: >- + /p:HostConfiguration=Release -priority1 -tree:JIT/CodeGenBringUpTests -tree:JIT/Generics @@ -68,3 +78,6 @@ jobs: -tree:reflection extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml + parameters: + liveLibrariesBuildConfig: Release + readyToRun: ${{ parameters.readyToRun }} diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index e58e2d50c0b2af..f95dc3c4fb4a82 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -174,6 +174,14 @@ extends: alwaysRun: ${{ variables.isRollingBuild }} useHelixMonitor: ${{ variables.enableHelixJobMonitor }} + - template: /eng/pipelines/common/templates/wasi-wasm-coreclr-runtime-tests.yml + parameters: + platforms: + - wasi_wasm + readyToRun: true + alwaysRun: ${{ variables.isRollingBuild }} + useHelixMonitor: ${{ variables.enableHelixJobMonitor }} + # CoreCLR library-test smoke leg run on wasmtime via Helix # (wasi_wasm queue). Starts with a single-library smoke set; see # https://github.com/dotnet/runtime/issues/130129. diff --git a/eng/testing/wasi-r2r-provisioning.targets b/eng/testing/wasi-r2r-provisioning.targets new file mode 100644 index 00000000000000..3efe9381761423 --- /dev/null +++ b/eng/testing/wasi-r2r-provisioning.targets @@ -0,0 +1,47 @@ + + + <_WasiR2RToolOS Condition="'$(HostOS)' == 'linux'">linux + <_WasiR2RToolOS Condition="'$(HostOS)' == 'osx'">macos + <_WasiR2RToolArch Condition="'$(BuildArchitecture)' == 'x64'">x86_64 + <_WasiR2RToolArch Condition="'$(BuildArchitecture)' == 'arm64'">aarch64 + <_BinaryenArch>$(_WasiR2RToolArch) + <_BinaryenArch Condition="'$(HostOS)' == 'osx' and '$(BuildArchitecture)' == 'arm64'">arm64 + <_BinaryenArchiveName>binaryen-version_$(BinaryenVersion)-$(_BinaryenArch)-$(_WasiR2RToolOS) + <_WasmToolsArchiveName>wasm-tools-$(WasmToolsVersion)-$(_WasiR2RToolArch)-$(_WasiR2RToolOS) + + + + + + + + + + + + diff --git a/eng/wasm/PruneWasmToolCache.proj b/eng/wasm/PruneWasmToolCache.proj index 0febac1c46ef72..8347bf5d1552f6 100644 --- a/eng/wasm/PruneWasmToolCache.proj +++ b/eng/wasm/PruneWasmToolCache.proj @@ -29,6 +29,7 @@ <_WasmToolInUseStamp Include="$(EmscriptenSdkStampFile);$(WasiSdkStampFile);$(WasmtimeStampFile)" /> + <_WasmToolInUseStamp Include="$(BinaryenStampFile);$(WasmToolsStampFile)" /> <_WasmToolInUseStamp Include="$(ChromeStampFile);$(ChromeDriverStampFile)" Condition="'$(ChromeVersion)' != ''" /> <_WasmToolInUseStamp Include="$(V8StampFile)" Condition="'$(V8Version)' != ''" /> <_WasmToolInUseStamp Include="$(FirefoxStampFile);$(GeckoDriverStampFile)" Condition="'$(FirefoxRevision)' != ''" /> diff --git a/eng/wasm/WasmToolCache.props b/eng/wasm/WasmToolCache.props index 92021e914890bf..49a39f2bbe9561 100644 --- a/eng/wasm/WasmToolCache.props +++ b/eng/wasm/WasmToolCache.props @@ -1,6 +1,6 @@ @@ -61,6 +61,12 @@ $([System.IO.File]::ReadAllText('$(_WasiSdkVersionFile)').Trim()) + + + 130 + 1.253.0 + + @@ -73,5 +79,11 @@ $([MSBuild]::NormalizeDirectory('$(WasmToolCacheDir)', 'wasmtime', '$(WasmtimeVersion)-$(WasmToolHostRid)')) $([MSBuild]::NormalizePath('$(WasmToolCacheDir)', 'wasmtime', '$(WasmtimeVersion)-$(WasmToolHostRid).complete')) + + $([MSBuild]::NormalizeDirectory('$(WasmToolCacheDir)', 'binaryen', '$(BinaryenVersion)-$(WasmToolHostRid)')) + $([MSBuild]::NormalizePath('$(WasmToolCacheDir)', 'binaryen', '$(BinaryenVersion)-$(WasmToolHostRid).complete')) + + $([MSBuild]::NormalizeDirectory('$(WasmToolCacheDir)', 'wasm-tools', '$(WasmToolsVersion)-$(WasmToolHostRid)')) + $([MSBuild]::NormalizePath('$(WasmToolCacheDir)', 'wasm-tools', '$(WasmToolsVersion)-$(WasmToolHostRid).complete')) diff --git a/src/coreclr/hosts/corerun/CMakeLists.txt b/src/coreclr/hosts/corerun/CMakeLists.txt index 4965996f4b7eb3..f58546186e8faf 100644 --- a/src/coreclr/hosts/corerun/CMakeLists.txt +++ b/src/coreclr/hosts/corerun/CMakeLists.txt @@ -4,6 +4,16 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON) set(CORERUN_IN_BROWSER 0) +# Link corerun so a composite ReadyToRun image can be spliced into it afterwards (see +# docs/workflow/building/coreclr/wasi-r2r.md). ON by default because corerun is the host this work is developed and +# tested against. Turning it OFF drops the table reservation and shrinks the probe's staging buffer +# to a stub, for a WASI corerun that will never be spliced. +option(CORERUN_WASI_COMPOSITE_R2R "Reserve table slots and export the globals a spliced R2R composite needs" ON) +# Table slots 1..N-1 are reserved for the composite's ACTIVE element segment, which the engine installs +# at instantiation, so the table must already be large enough. Reserve by the composite's FUNCTION +# count, not its assembly count; compose-r2r.py checks this and names the value needed. +set(CORERUN_WASI_R2R_TABLE_BASE "65537" CACHE STRING "First table slot for corerun's own address-taken functions") + if(CLR_CMAKE_HOST_WIN32) add_definitions(-DFX_VER_INTERNALNAME_STR=corerun.exe) else() @@ -157,7 +167,36 @@ else() -Wl,-z,stack-size=8388608 -Wl,--initial-memory=134217728 -Wl,--max-memory=4294967296 - "-Wl,--component-type,${_wasi_http_world_wit}") + "-Wl,--component-type,${_wasi_http_world_wit}" + # Force-root the cDAC contract descriptor getter: its object has no live + # references (the getter is the only one, and DATA symbols aren't auto-exported), + # so without an explicit --export root wasm-ld's --gc-sections drops the whole + # object and the descriptor never reaches the final module. Mirrors the native + # `-Wl,-u,DotNetRuntimeContractDescriptor` used on other non-browser targets. + -Wl,--export=GetDotNetRuntimeContractDescriptor + # [wasi] Export the runtime-owned async continuation global (#131167, helpers.cpp) so the + # offline R2R merge can wire each composite's webcil.asyncContinuation import to this same + # global (the interp<->R2R accessors use it), and so --gc-sections doesn't drop it. + -Wl,--export=__async_continuation) + + if (CORERUN_WASI_COMPOSITE_R2R) + # Supply the rest of a spliced composite's imports at link time. --table-base reserves + # the low table slots for the composite's ACTIVE element segment and moves corerun's own + # address-taken functions above it; the table stays fixed-size so the engine can still + # validate it statically. These must match the flags in + # src/mono/wasi/build/WasiApp.CoreCLR.targets, which links the per-app host with the + # same probe. + target_link_options(corerun PRIVATE + -Wl,--table-base=${CORERUN_WASI_R2R_TABLE_BASE} + -Wl,--export-table + -Wl,--export=__stack_pointer + -Wl,--export=__coreclr_wasm_rtlrestorecontext_tag) + else() + # No splice for this host: keep the probe compiled (it simply finds no image and + # declines) but drop its staging buffer, which is otherwise WASI_R2R_IMAGE_CAP of BSS + # taken out of --initial-memory whether or not R2R is ever used. + target_compile_definitions(corerun PRIVATE WASI_R2R_IMAGE_CAP=64u) + endif() endif() if (CORERUN_IN_BROWSER) diff --git a/src/coreclr/hosts/corerun/corerun.cpp b/src/coreclr/hosts/corerun/corerun.cpp index 49908c2675ac74..93184a93628d74 100644 --- a/src/coreclr/hosts/corerun/corerun.cpp +++ b/src/coreclr/hosts/corerun/corerun.cpp @@ -308,6 +308,10 @@ static char* s_core_root_path = nullptr; extern "C" bool BrowserHost_ExternalAssemblyProbe(const char* pathPtr, /*out*/ void **outDataStartPtr, /*out*/ int64_t* outSize); #endif // TARGET_BROWSER +// WASI R2R external-assembly probe, shared with the wasihost corehost (libWasiHost.a) +// so both hosts serve R2R identically. See wasi_r2r_probe.hpp. +#include "wasi_r2r_probe.hpp" + static bool HOST_CONTRACT_CALLTYPE get_native_code_data( const host_runtime_contract_native_code_context* context, host_runtime_contract_native_code_data* data) @@ -376,6 +380,14 @@ static bool HOST_CONTRACT_CALLTYPE external_assembly_probe( if (pos != NULL) name = pos + 1; +#ifdef TARGET_WASI + { + const char* const r2r_dirs[] = { s_core_libs_path, s_core_root_path }; + if (wasi_r2r::WasiStaticR2RProbe(name, r2r_dirs, 2, data_start, size)) + return true; + } +#endif // TARGET_WASI + // Try to map the file from our known app assembly paths for (const char* dir : { s_core_libs_path, s_core_root_path }) { diff --git a/src/coreclr/hosts/corerun/wasi_r2r_probe.hpp b/src/coreclr/hosts/corerun/wasi_r2r_probe.hpp new file mode 100644 index 00000000000000..25ba71b610ebdd --- /dev/null +++ b/src/coreclr/hosts/corerun/wasi_r2r_probe.hpp @@ -0,0 +1,327 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// Statically-composed WASI R2R external-assembly probe, shared by every CoreCLR-WASI host +// (the standalone corerun executable and the per-app-linked wasihost corehost / libWasiHost.a). The +// probe is a host_runtime_contract::external_assembly_probe callback: the runtime calls out to it to +// obtain the composite R2R webcil image and the per-assembly stubs. Keeping it here (rather than in a +// single host) means both hosts serve R2R identically instead of one silently falling back to interp. +// +// src/mono/wasi/build/compose-r2r.py populates the image for in-tree publishing and runtime tests. +// Both hosts must be linked with the flags that supply a composite's imports -- see +// CORERUN_WASI_COMPOSITE_R2R in corerun/CMakeLists.txt and WasiEnableCompositeR2R in +// WasiApp.CoreCLR.targets. Without them this probe compiles but can never be satisfied. +// +// Requires corerun.hpp to be included first (for pal::try_map_file_readonly). Include exactly once per +// host translation unit; the internal-linkage buffer/functions then give one instance per host binary. + +#ifndef WASI_R2R_PROBE_HPP +#define WASI_R2R_PROBE_HPP + +#ifdef TARGET_WASI + +#include +#include +#include +#include +#include + +namespace wasi_r2r +{ +// A crossgen2-produced R2R webcil image is merged into the host module post-link (its native +// functions land in the shared indirect function table and its webcil payload/metadata is written +// into g_wasi_r2r_image by the offline merge's active data segment at this buffer's address == the +// composite image's imageBase). The runtime then finds the R2R webcil via this probe, exactly the way +// the browser host does via BrowserHost_ExternalAssemblyProbe. +// +// Standalone corerun uses the fixed development buffer below. The per-app host declares the symbols +// external instead: its publish builds a strong buffer definition sized exactly to the composite, +// while the host archive carries a 64-byte weak fallback for non-R2R apps. +#ifndef WASI_R2R_IMAGE_CAP +#define WASI_R2R_IMAGE_CAP (16u * 1024u * 1024u) +#endif + +#ifdef WASI_R2R_EXTERNAL_IMAGE_BUFFER +extern "C" uint8_t g_wasi_r2r_image[]; +extern "C" uint32_t g_wasi_r2r_image_cap; +#else +alignas(16) static uint8_t g_wasi_r2r_image[WASI_R2R_IMAGE_CAP]; +static constexpr uint32_t g_wasi_r2r_image_cap = WASI_R2R_IMAGE_CAP; +#endif + +// The table index at which the composite's functions are installed. Under the reservation model the +// host is linked with `-Wl,--table-base=`, which moves corerun's own address-taken functions up +// to start at N+1 and leaves slots 1..N free, so the composite always sits at base 1 regardless of +// its size. This MUST match the `__table_base` global supplied to the merge (see docs/design/mono/webcil.md); +// the two are a coupled constant and a mismatch is silent -- see the patch in WasiStaticR2RProbe. +#ifndef WASI_R2R_TABLE_BASE +#define WASI_R2R_TABLE_BASE (1u) +#endif +// Header version 1 adds TableBase to the 28-byte version 0 header. +#define WEBCIL_HEADER_V0_SIZE (28u) +#define WEBCIL_HEADER_V1_SIZE (32u) +#define WEBCIL_SECTION_HEADER_SIZE (16u) +#define WEBCIL_VERSION_MAJOR_OFFSET (4u) +#define WEBCIL_TABLE_BASE_OFFSET (28u) + +// The composite native image's bundle-relative file name (the ownerCompositeExecutable named by each +// per-assembly stub). The runtime asks for this via NativeImage::Open -> external_assembly_probe. +#ifndef WASI_R2R_COMPOSITE_NAME +#define WASI_R2R_COMPOSITE_NAME "composite-r2r.wasm" +#endif + +static size_t WasiWebcilHeaderSize(const uint8_t* p, size_t len) +{ + if (len < WEBCIL_HEADER_V0_SIZE) + return 0; + + uint16_t versionMajor; + memcpy(&versionMajor, p + WEBCIL_VERSION_MAJOR_OFFSET, sizeof(versionMajor)); + return versionMajor >= 1 ? WEBCIL_HEADER_V1_SIZE : WEBCIL_HEADER_V0_SIZE; +} + +// Compute the exact WbIL payload size from its self-describing header - no baked constant needed. +// WebcilHeader_1 (32 bytes): Id[4] 'WbIL', VersionMajor u16, VersionMinor u16, CoffSections u16, +// Reserved0 u16, PeCliHeaderRva u32, PeCliHeaderSize u32, PeDebugRva u32, PeDebugSize u32, TableBase u32. +// Followed by CoffSections * WebcilSectionHeader{VirtualSize, VirtualAddress, SizeOfRawData, PointerToRawData}. +// The payload extent is the maximum (PointerToRawData + SizeOfRawData) across all sections. +// +// Every field here comes from an image this host did not produce, so bounds and overflow are checked +// rather than assumed: a wrapped sum would yield a SMALL extent that passes the cap check below and +// hands the runtime a truncated image. +static int64_t WasiWebcilPayloadSize(const uint8_t* p, size_t len) +{ + size_t headerSize = WasiWebcilHeaderSize(p, len); + if (headerSize == 0 || headerSize > len) + return 0; + + if (p[0] != 'W' || p[1] != 'b' || p[2] != 'I' || p[3] != 'L') + return 0; + + uint16_t coffSections; + memcpy(&coffSections, p + 8, sizeof(coffSections)); + + // Section headers must fit entirely within the buffer. + if ((len - headerSize) / WEBCIL_SECTION_HEADER_SIZE < coffSections) + return 0; + + const uint8_t* sec = p + headerSize; + uint32_t maxEnd = 0; + for (uint16_t i = 0; i < coffSections; i++) + { + uint32_t sizeOfRawData; + uint32_t pointerToRawData; + memcpy(&sizeOfRawData, sec + 8, sizeof(sizeOfRawData)); + memcpy(&pointerToRawData, sec + 12, sizeof(pointerToRawData)); + + // Reject rather than wrap: UINT32_MAX - a < b <=> a + b would overflow. + if (UINT32_MAX - pointerToRawData < sizeOfRawData) + return 0; + + uint32_t end = pointerToRawData + sizeOfRawData; + if (end > maxEnd) + maxEnd = end; + sec += WEBCIL_SECTION_HEADER_SIZE; + } + return (int64_t)maxEnd; +} + +// Minimal LEB128 reader for parsing a wasm binary's Data section. Returns false on a truncated or +// over-long encoding rather than shifting past the width of the result (which would be UB). +static bool wasi_read_uleb(const uint8_t* p, size_t len, size_t* pos, uint64_t* value) +{ + uint64_t result = 0; + int shift = 0; + while (*pos < len) + { + uint8_t b = p[(*pos)++]; + if (shift >= 64) + return false; // over-long encoding + result |= (uint64_t)(b & 0x7f) << shift; + if ((b & 0x80) == 0) + { + *value = result; + return true; + } + shift += 7; + } + return false; // ran off the end without a terminating byte +} + +// Extract the raw WbIL webcil payload (passive data segment index 1) from a wasm-wrapped-webcil stub +// on disk. The stub's tableBase field (WEBCIL_TABLE_BASE_OFFSET) is authoritative: the offline merge +// step patches it to the composite's merge-time table base, so this host trusts the on-disk value +// rather than injecting a baked constant. +// Mirrors what the browser JS loader's getWebcilPayload does, but purely in native code (no instantiation). +// +// On success the file mapping is deliberately RETAINED and *data_start points into it: the runtime +// takes ownership of neither (ProbeExtensionResult::External never frees), so copying to a malloc'd +// buffer would leak the copy on top of the mapping. Every failure path unmaps. +// +// The stub is untrusted input, so each length read is validated against the remaining extent before +// it is used to advance or copy. +static bool WasiExtractStubPayload(const char* wasmPath, void** data_start, int64_t* size) +{ + void* filedata = nullptr; int64_t filesize = 0; + if (!pal::try_map_file_readonly(wasmPath, &filedata, &filesize)) + return false; + + const uint8_t* p = (const uint8_t*)filedata; + size_t len = (size_t)filesize; + bool ok = false; + if (len >= 8 && p[0] == 0x00 && p[1] == 0x61 && p[2] == 0x73 && p[3] == 0x6d) + { + size_t pos = 8; + while (pos < len) + { + uint8_t secId = p[pos++]; + uint64_t secSize; + if (!wasi_read_uleb(p, len, &pos, &secSize)) + break; + // len - pos cannot underflow (pos <= len) and avoids overflowing pos + secSize, which + // wraps on wasm32 where size_t is 32-bit. + if (secSize > (uint64_t)(len - pos)) + break; + size_t secEnd = pos + (size_t)secSize; + if (secId == 11) // Data section + { + size_t q = pos; + uint64_t segCount; + if (!wasi_read_uleb(p, secEnd, &q, &segCount)) + break; + for (uint64_t s = 0; s < segCount && q < secEnd; s++) + { + uint64_t mode; + if (!wasi_read_uleb(p, secEnd, &q, &mode)) + break; + // Only passive segments (mode 1) are used by the webcil wrapper. A composite's + // payload segment is ACTIVE, so this also declines a composite handed here by + // mistake rather than misreading its offset expression as segment data. + if (mode != 1) { break; } + uint64_t dlen; + if (!wasi_read_uleb(p, secEnd, &q, &dlen)) + break; + if (dlen > (uint64_t)(secEnd - q)) + break; // segment claims more bytes than the section holds + size_t dstart = q; + q += (size_t)dlen; + if (s == 1) // segment[1] == the WbIL payload + { + // Validate rather than assume: if the wrapper's segment layout ever changes, + // fail loudly here instead of handing the runtime a non-webcil buffer. + if (dlen >= 4 && memcmp(p + dstart, "WbIL", 4) == 0) + { + *data_start = (void*)(p + dstart); + *size = (int64_t)dlen; + ok = true; + } + break; + } + } + break; + } + pos = secEnd; + } + } + if (!ok) + munmap(filedata, (size_t)filesize); + return ok; +} + +// The external-assembly R2R probe: serves the composite webcil from the baked buffer and each managed +// assembly's per-assembly stub from "/comp/.wasm" on disk, searching the supplied dirs (each +// expected to carry a trailing path delimiter). Returns false for anything it does not provide, letting +// the caller fall back to its normal assembly load. +static bool WasiStaticR2RProbe(const char* name, const char* const* dirs, size_t ndirs, void** data_start, int64_t* size) +{ + // The composite native image itself: return the merged composite payload at imageBase. Its size is + // read from the self-describing WbIL header (no baked constant), and validated against the buffer cap. + if (strcmp(name, WASI_R2R_COMPOSITE_NAME) == 0) + { + int64_t payloadSize = WasiWebcilPayloadSize(&g_wasi_r2r_image[0], g_wasi_r2r_image_cap); + if (payloadSize <= 0 || (size_t)payloadSize > g_wasi_r2r_image_cap) + return false; // buffer not populated, or composite payload exceeds the cap + + // A current self-installing image patches its own TableBase from the composition shim's start + // function. Older images predate patchWebcilHeader, so retain the native fallback when the + // field is still zero. WebcilDecoder treats an unwritten zero as a valid base and would + // otherwise shift every R2R function index to the wrong table slot. + // + // NOTE: the cap test above cannot protect this buffer -- the engine installs the segment before any + // host code runs, so an over-cap payload has already overwritten whatever follows by the time we look. + // The enforceable check is at build time; compose-r2r.py compares the payload size against the cap. + uint8_t* hdr = &g_wasi_r2r_image[0]; + if (WasiWebcilHeaderSize(hdr, (size_t)payloadSize) >= WEBCIL_HEADER_V1_SIZE) + { + uint32_t existingTableBase; + memcpy(&existingTableBase, hdr + WEBCIL_TABLE_BASE_OFFSET, sizeof(existingTableBase)); + if (existingTableBase == 0) + { + uint32_t tableBase = WASI_R2R_TABLE_BASE; + memcpy(hdr + WEBCIL_TABLE_BASE_OFFSET, &tableBase, sizeof(tableBase)); + } + } + + *data_start = &g_wasi_r2r_image[0]; + *size = payloadSize; + return true; + } + + // A managed assembly: return its per-assembly stub payload (extracted from .wasm on disk). + // The stub carries the assembly metadata + the R2R header naming the composite, which drives the + // runtime to then request WASI_R2R_COMPOSITE_NAME above. + size_t nlen = strlen(name); + if (nlen > 4 && strcmp(name + nlen - 4, ".dll") == 0) + { + char stub[512]; + for (size_t i = 0; i < ndirs; i++) + { + const char* dir = dirs[i]; + if (dir == nullptr) continue; + // Build "/comp/.wasm" + snprintf(stub, sizeof(stub), "%scomp/%.*s.wasm", dir, (int)(nlen - 4), name); + if (WasiExtractStubPayload(stub, data_start, size)) + { + return true; + } + } + } + return false; +} + +} // namespace wasi_r2r + +// Exported so the offline merge step can discover the buffer's address and wire it to the R2R image's +// __memory_base global. Defined outside the namespace with C linkage so the export name is exactly +// "wasi_r2r_image_base" (the merge step targets this symbol). +// +// This header is included once in each host binary. Keeping the exported accessor here ensures the +// linker roots the selected fixed or per-app buffer and gives the composition step a stable anchor. +extern "C" __attribute__((export_name("wasi_r2r_image_base"))) uint32_t wasi_r2r_image_base(void) +{ + return (uint32_t)(uintptr_t)&wasi_r2r::g_wasi_r2r_image[0]; +} + +// The staging buffer's capacity and the table slot the composite installs at, exported for the same +// reason as the base: the splice must not carry its own copy of either. The host owns these values; +// compose-r2r.py reads them out of the linked binary and validates the composite +// against them, so a mismatch is a build-time error instead of a wrong-function dispatch at runtime. +#ifdef WASI_R2R_EXTERNAL_IMAGE_BUFFER +#define WASI_R2R_IMAGE_CAP_WEAK __attribute__((weak)) +#else +#define WASI_R2R_IMAGE_CAP_WEAK +#endif +extern "C" WASI_R2R_IMAGE_CAP_WEAK __attribute__((export_name("wasi_r2r_image_cap"))) uint32_t wasi_r2r_image_cap(void) +{ + return wasi_r2r::g_wasi_r2r_image_cap; +} +#undef WASI_R2R_IMAGE_CAP_WEAK + +extern "C" __attribute__((export_name("wasi_r2r_table_base"))) uint32_t wasi_r2r_table_base(void) +{ + return (uint32_t)WASI_R2R_TABLE_BASE; +} + +#endif // TARGET_WASI + +#endif // WASI_R2R_PROBE_HPP diff --git a/src/coreclr/hosts/corerun/wasm/libCorerun.js b/src/coreclr/hosts/corerun/wasm/libCorerun.js index 4d0d4f80a275e8..4227cad47d2112 100644 --- a/src/coreclr/hosts/corerun/wasm/libCorerun.js +++ b/src/coreclr/hosts/corerun/wasm/libCorerun.js @@ -245,13 +245,13 @@ function libCoreRunFactory() { wasmInstance = new WebAssembly.Instance(wasmModule, { webcil: { memory: wasmMemory, - stackPointer: wasmExports.__stack_pointer, - rtlRestoreContextTag: wasmExports.__coreclr_wasm_rtlrestorecontext_tag, - table: wasmTable, - tableBase: new WebAssembly.Global({ value: "i32", mutable: false }, tableStartIndex), - imageBase: new WebAssembly.Global({ value: "i32", mutable: false }, payloadPtr), + __stack_pointer: wasmExports.__stack_pointer, + __coreclr_wasm_rtlrestorecontext_tag: wasmExports.__coreclr_wasm_rtlrestorecontext_tag, + __indirect_function_table: wasmTable, + __table_base: new WebAssembly.Global({ value: "i32", mutable: false }, tableStartIndex), + __memory_base: new WebAssembly.Global({ value: "i32", mutable: false }, payloadPtr), // Runtime-async continuation return value, shared with the runtime module. - asyncContinuation: wasmExports.__async_continuation + __async_continuation: wasmExports.__async_continuation } }); } catch (e) { @@ -267,10 +267,21 @@ function libCoreRunFactory() { throw new Error(`Unsupported Webcil version: ${webcilVersion}`); } - wasmInstance.exports.getWebcilPayload(payloadPtr, payloadSize); - if (tableSize > 0) { - wasmInstance.exports.fillWebcilTable(); + // Two image shapes reach this point. A component stub carries its payload and table in + // passive segments and hands them over via getWebcilPayload/fillWebcilTable. A composite + // uses active segments, so the engine has already installed both by the time the instance + // exists, and only the header's tableBase field is left to write. Feature-detect rather + // than assume: calling getWebcilPayload on a composite would trap, because memory.init + // against an active (hence dropped) segment is out of bounds. + if (typeof (wasmInstance.exports.patchWebcilHeader) === "function") { + wasmInstance.exports.patchWebcilHeader(payloadPtr, payloadSize); + } else { + wasmInstance.exports.getWebcilPayload(payloadPtr, payloadSize); + if (tableSize > 0) { + wasmInstance.exports.fillWebcilTable(); + } } + HEAPU32[outDataStartPtr >>> 2 >>> 0] = payloadPtr; HEAPU32[outSize >>> 2 >>> 0] = payloadSize; HEAPU32[(outSize + 4) >>> 2 >>> 0] = 0; diff --git a/src/coreclr/tools/Common/Compiler/DependencyAnalysis/WasmWellKnownGlobalSymbolNode.cs b/src/coreclr/tools/Common/Compiler/DependencyAnalysis/WasmWellKnownGlobalSymbolNode.cs index 7d0560a9c96302..1e4ccf07a5dc8a 100644 --- a/src/coreclr/tools/Common/Compiler/DependencyAnalysis/WasmWellKnownGlobalSymbolNode.cs +++ b/src/coreclr/tools/Common/Compiler/DependencyAnalysis/WasmWellKnownGlobalSymbolNode.cs @@ -11,19 +11,26 @@ namespace ILCompiler.DependencyAnalysis /// crossgen2/R2R resolves it back to the fixed index defined in the WebCIL format, while a relocatable /// NativeAOT object emits it as an undefined imported global for wasm-ld to resolve. /// + /// + /// The names deliberately match what wasm-ld uses, so that a composite's imports resolve + /// against the host's exports by name and wasm-merge needs no renaming step. + /// __stack_pointer and __indirect_function_table come from the linker directly; + /// __memory_base and __table_base are wasm-ld's PIC names for exactly these two + /// quantities (where a module's data and table slice begin) and must be defined and exported by + /// the host, since a non-PIC main module does not produce them on its own. + /// + /// Those last two belong to the emscripten/wasm-ld dynamic linking ABI, where they carry a + /// per-side-module meaning. Reusing them is safe only while the host is a non-PIC main module. + /// Building the host with -sMAIN_MODULE would give the linker its own definitions of both + /// and collide with these; that would require picking runtime-specific names instead. + /// + /// public class WasmWellKnownGlobalSymbolNode(string symbolName) : ExternDataSymbolNode(new Utf8String(symbolName)) { -#if READYTORUN - public const string StackPointerName = "stackPointer"; - public const string ImageBaseName = "imageBase"; - public const string TableBaseName = "tableBase"; - public const string AsyncContinuationName = "asyncContinuation"; -#else public const string StackPointerName = "__stack_pointer"; public const string ImageBaseName = "__memory_base"; public const string TableBaseName = "__table_base"; public const string AsyncContinuationName = "__async_continuation"; -#endif public override int ClassCode => 0x79046cf9; diff --git a/src/coreclr/tools/Common/Compiler/ObjectWriter/Wasm/WasmSection.cs b/src/coreclr/tools/Common/Compiler/ObjectWriter/Wasm/WasmSection.cs index c3eeb12f83d525..0b93009dcbdcbd 100644 --- a/src/coreclr/tools/Common/Compiler/ObjectWriter/Wasm/WasmSection.cs +++ b/src/coreclr/tools/Common/Compiler/ObjectWriter/Wasm/WasmSection.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.IO; using System.Numerics; +using ILCompiler.ObjectWriter.WasmInstructions; using Internal.Text; using Internal.TypeSystem; @@ -342,19 +343,51 @@ protected override void WriteEntryCore(SectionWriter writer, WasmExport entry) } } - internal sealed class WasmElementSection : WasmSection> + /// + /// A funcref element segment targeting table 0. + /// + /// + /// When is the segment is emitted as passive + /// (flag 1), which requires a table.init to install it. Otherwise it is emitted as + /// active (flag 0) with as its offset constant expression, and the + /// engine installs it at instantiation. + /// + internal readonly struct WasmElementSegment + { + public ReadOnlyMemory FunctionIndices { get; } + public WasmInstructionGroup OffsetExpr { get; } + + public WasmElementSegment(ReadOnlyMemory functionIndices, WasmInstructionGroup offsetExpr = null) + { + FunctionIndices = functionIndices; + OffsetExpr = offsetExpr; + } + } + + internal sealed class WasmElementSection : WasmSection { public WasmElementSection(Stream stream, Utf8String name, int sectionIndex) : base(WasmSectionType.Element, stream, name, sectionIndex) { } - protected override void WriteEntryCore(SectionWriter writer, ReadOnlyMemory entry) + protected override void WriteEntryCore(SectionWriter writer, WasmElementSegment entry) { - ReadOnlySpan functionIndices = entry.Span; + ReadOnlySpan functionIndices = entry.FunctionIndices.Span; + + if (entry.OffsetExpr is not null) + { + // Active element segment, table 0. Flag 0 implies both the table index and the + // funcref element type, so no element-type byte follows. + writer.WriteByte(0); + WriteEncodable(writer, entry.OffsetExpr); + } + else + { + writer.WriteByte(1); // Passive element segment + writer.WriteByte(0); // element type: ref func + } - writer.WriteByte(1); // Passive element segment - writer.WriteByte(0); // element type: ref func writer.WriteULEB128((ulong)functionIndices.Length); foreach (int functionIndex in functionIndices) { diff --git a/src/coreclr/tools/Common/Compiler/ObjectWriter/Wasm/WebcilPayloadDataSegment.cs b/src/coreclr/tools/Common/Compiler/ObjectWriter/Wasm/WebcilPayloadDataSegment.cs index 50abf531c68e98..53a9bb0c43feff 100644 --- a/src/coreclr/tools/Common/Compiler/ObjectWriter/Wasm/WebcilPayloadDataSegment.cs +++ b/src/coreclr/tools/Common/Compiler/ObjectWriter/Wasm/WebcilPayloadDataSegment.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics; using System.IO; +using ILCompiler.ObjectWriter.WasmInstructions; using Internal.TypeSystem; using Microsoft.NET.WebAssembly.Webcil; @@ -12,19 +13,22 @@ namespace ILCompiler.ObjectWriter /// /// The data segment of Webcil modules that contains the Webcil payload composed of WebcilSections. /// - internal sealed class WebcilPayloadDataSegment : IWasmDataSegment + internal sealed class WebcilPayloadDataSegment : IWasmActiveDataSegment { private readonly WebcilHeader _header; private readonly WebcilSection[] _sections; + private readonly WasmInstructionGroup _offsetExpr; private readonly int _alignment; private int _paddingBytesCount; public WebcilPayloadDataSegment( WebcilHeader header, - WebcilSection[] sections) + WebcilSection[] sections, + WasmInstructionGroup offsetExpr = null) { _header = header; _sections = sections; + _offsetExpr = offsetExpr; _alignment = WebCilObjectWriter.WebcilSectionAlignment; foreach (WebcilSection section in sections) { @@ -33,9 +37,10 @@ public WebcilPayloadDataSegment( } public int HeaderSize => - WasmDataSegmentEncoding.GetHeaderSize(WasmDataSegmentType.Passive, initExpr: null); + WasmDataSegmentEncoding.GetHeaderSize(SegmentType, _offsetExpr); public int FileAlignment => _alignment; + public int MemoryAlignment => _alignment; private int RawContentSize { @@ -56,7 +61,8 @@ private int RawContentSize public int ContentSize => checked(RawContentSize + _paddingBytesCount); - public WasmDataSegmentType SegmentType => WasmDataSegmentType.Passive; + public WasmDataSegmentType SegmentType => + _offsetExpr is null ? WasmDataSegmentType.Passive : WasmDataSegmentType.Active; public int EncodeSize() => HeaderSize + ContentSize; @@ -65,8 +71,8 @@ public int EmitToStream(Stream outputFileStream) Span headerBuffer = stackalloc byte[HeaderSize]; int headerSize = WasmDataSegmentEncoding.EncodeHeader( headerBuffer, - WasmDataSegmentType.Passive, - initExpr: null, + SegmentType, + _offsetExpr, ContentSize); Debug.Assert(headerSize == HeaderSize); outputFileStream.Write(headerBuffer); @@ -109,5 +115,12 @@ public int GetMemoryAddressOfOffset(int offsetInSegment) Debug.Assert(offsetInSegment >= 0 && offsetInSegment <= RawContentSize); return offsetInSegment; } + + public void SetMemoryOffset(int offset) + { + // The payload is the only active segment and is based at the host-supplied image base. + // Its section RVAs remain relative to that base, not to an absolute linear-memory address. + Debug.Assert(offset == 0); + } } } diff --git a/src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs b/src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs index 2d9bb6a8f946d2..4ef47fc9fb96bb 100644 --- a/src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs +++ b/src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs @@ -264,12 +264,12 @@ private protected void WriteMemoryExport(string name, int memoryIndex) => private protected void WriteGlobalExport(string name, int globalIndex) => WriteExport(name, WasmExportKind.Global, globalIndex); - private protected void WriteElementSegment(ReadOnlyMemory functionIndices) + private protected void WriteElementSegment(ReadOnlyMemory functionIndices, WasmInstructionGroup offsetExpr = null) { WasmElementSection section = GetOrCreateSection( WasmObjectNodeSection.ElementSection, out SectionWriter writer); - section.WriteEntry(writer, functionIndices); + section.WriteEntry(writer, new WasmElementSegment(functionIndices, offsetExpr)); } private protected SectionDataEmitter GetOrCreateSection( diff --git a/src/coreclr/tools/Common/Compiler/ObjectWriter/WebCilObjectWriter.cs b/src/coreclr/tools/Common/Compiler/ObjectWriter/WebCilObjectWriter.cs index 9c972a705d6c7d..fe001e25b0a68a 100644 --- a/src/coreclr/tools/Common/Compiler/ObjectWriter/WebCilObjectWriter.cs +++ b/src/coreclr/tools/Common/Compiler/ObjectWriter/WebCilObjectWriter.cs @@ -125,6 +125,30 @@ WasmInstructionGroup GetImageFunctionPointerBaseOffset(int offset) ] ); + /// + /// Counterpart of for a self-installing image. Its payload lives in an + /// active data segment, so the engine has already copied it to imageBase and a + /// memory.init would in fact trap: an active segment is implicitly dropped at + /// instantiation. All that remains is the header's tableBase field, which the runtime + /// reads from the mapped image rather than from a Wasm global + /// (WebcilDecoder::GetTableBaseOffset), and which returns 0 when unwritten - silently + /// shifting every R2R function index by tableBase. Both hosts must call this after + /// instantiation. + /// + static WasmFunctionBody PatchWebcilHeader = new WasmFunctionBody( + new WasmFuncType(new([WasmValueType.I32, WasmValueType.I32]), new([])), // (func ($d i32) ($n i32)) + [ + Local.Get(1), + I32.Const(32), + I32.Ge_s, + Block.If(WasmBlockType.Empty), + Local.Get(0), // (local.get $d) + Global.Get(WebCilObjectWriter.TableBaseGlobalIndex), // (global.get $tableBase) + I32.Store((ulong)WebcilEncoder.TableBaseOffset), // i32.store offset=TableBaseOffset + Block.End + ] + ); + private long ResolveSymbolRVA(WebcilSection[] sections, SymbolDefinition definition) { for (int i = 0; i < sections.Length; i++) @@ -242,12 +266,47 @@ private void WriteDataCountSection() writer.WriteULEB128(NumDataSegments); // number of data segments } + /// + /// Whether this image installs its own payload and function table via active segments. + /// + /// + /// True for any image that carries code - a composite or a single-assembly R2R image - since + /// the host instantiates those and the engine can apply the segments. False for a per-assembly + /// component forwarding stub, which must keep its payload passive: a stub is not necessarily + /// instantiated at all, and an offline host may instead parse it as a file and locate the + /// payload by passive data segment index, which an active segment would defeat. + /// + private bool IsSelfInstallingImage => !_nodeFactory.OptimizationFlags.IsComponentModule; + + /// Offset constant expression placing the payload segment at the host-supplied image base. + private static WasmInstructionGroup ImageBaseOffsetExpr => + new WasmInstructionGroup([Global.Get(ImageBaseGlobalIndex)]); + + /// Offset constant expression placing the function table slice at the host-supplied table base. + private static WasmInstructionGroup TableBaseOffsetExpr => + new WasmInstructionGroup([Global.Get(TableBaseGlobalIndex)]); + private protected override void EmitSectionsAndLayout() { - int totalMethodCount = MethodCount + 3; + // The stub set is image-kind dependent. A self-installing image installs both its payload and its + // function table via active segments, so it needs neither getWebcilPayload (whose + // memory.init would trap against a dropped active segment) nor fillWebcilTable (whose + // work the engine has already done); it needs only patchWebcilHeader, because the + // header's tableBase field is read from linear memory and cannot come from a segment. + // A component stub keeps all three: it is passive throughout and is installed by the host. + int stubCount = IsSelfInstallingImage ? 2 : 3; + int totalMethodCount = MethodCount + stubCount; InsertWasmStub(new Utf8String("getWebcilSize"), GetWebcilSize); - InsertWasmStub(new Utf8String("getWebcilPayload"), GetWebcilPayload); - InsertWasmStub(new Utf8String("fillWebcilTable"), FillWebcilTable(totalMethodCount)); + if (IsSelfInstallingImage) + { + InsertWasmStub(new Utf8String("patchWebcilHeader"), PatchWebcilHeader); + } + else + { + InsertWasmStub(new Utf8String("getWebcilPayload"), GetWebcilPayload); + InsertWasmStub(new Utf8String("fillWebcilTable"), FillWebcilTable(totalMethodCount)); + } + Debug.Assert(MethodCount == totalMethodCount); WriteDataCountSection(); @@ -343,7 +402,9 @@ private protected override void EmitObjectFile(Stream outputFileStream) webcilSections = _sections.Sections.OfType().ToArray(); WebcilHeader webcilHeader = LayoutWebcilPayload(webcilSections); ResolveWebcilSectionRelocations(webcilSections); - WebcilPayloadDataSegment webcilPayloadSegment = new(webcilHeader, webcilSections); + // Component stubs remain passive for hosts that extract their payload without instantiation. + WebcilPayloadDataSegment webcilPayloadSegment = new( + webcilHeader, webcilSections, IsSelfInstallingImage ? ImageBaseOffsetExpr : null); // Writing our memory import <- size of the webcil segment (for an accurate minimum size) WriteMemoryImport((ulong)webcilPayloadSegment.ContentSize); @@ -388,7 +449,9 @@ private protected override void EmitObjectFile(Stream outputFileStream) * Emit Webcil segment at end of file to support ReadyToRun ****************************************************************/ - // Create passive data segment for encoding the size of the webcil payload (size must fit in 32-bit uint) + // Passive data segment for the payload size; this is metadata for the host loader, which + // reads it straight out of the module bytes before instantiation in order to size its + // allocation, so it must stay passive in both image shapes. byte[] lengthBuffer = new byte[sizeof(uint) * 2]; BinaryPrimitives.WriteUInt32LittleEndian(lengthBuffer, (uint)webcilPayloadSegment.ContentSize); BinaryPrimitives.WriteUInt32LittleEndian(lengthBuffer.AsSpan().Slice(4), (uint)MethodCount); @@ -890,7 +953,14 @@ private void ResolveRelocations(int sectionIndex, Stream sectionStream, MemorySt #nullable disable public const int RtlRestoreContextTagIndex = 0; - private static readonly Utf8String RtlRestoreContextTagName = new("rtlRestoreContextTag"); + + /// + /// Import field names for the table and tag. These match the corresponding wasm-ld + /// exports (--export-table emits __indirect_function_table), so that a + /// composite merged into the host resolves by name with no renaming step. + /// + private const string IndirectFunctionTableName = "__indirect_function_table"; + private static readonly Utf8String RtlRestoreContextTagName = new("__coreclr_wasm_rtlrestorecontext_tag"); private static readonly WasmFuncType RtlRestoreContextTagSignature = new( new([]), @@ -911,7 +981,7 @@ private WasmImport[] CreateDefaultGlobalImports() new WasmImport("webcil", WasmWellKnownGlobalSymbolNode.ImageBaseName, import: new WasmGlobalImportType(WasmValueType.I32, WasmMutabilityType.Const), index: ImageBaseGlobalIndex), new WasmImport("webcil", WasmWellKnownGlobalSymbolNode.TableBaseName, import: new WasmGlobalImportType(WasmValueType.I32, WasmMutabilityType.Const), index: TableBaseGlobalIndex), new WasmImport("webcil", WasmWellKnownGlobalSymbolNode.AsyncContinuationName, import: new WasmGlobalImportType(WasmValueType.I32, WasmMutabilityType.Mut), index: AsyncContinuationGlobalIndex), - new WasmImport("webcil", "table", import: new WasmTableImportType(), index: 0), + new WasmImport("webcil", IndirectFunctionTableName, import: new WasmTableImportType(), index: 0), new WasmImport("webcil", RtlRestoreContextTagName.ToString(), import: new WasmTagImportType(rtlRestoreContextTagTypeIndex), index: RtlRestoreContextTagIndex), ]; } @@ -961,7 +1031,9 @@ private protected override void WriteElements() .Select(symbol => symbol.Index) .ToArray(); - WriteElementSegment(functionIndices); + // A self-installing image installs its table slice via an active segment at the host-supplied table + // base. A component stub stays passive; it has no table slice of its own to install. + WriteElementSegment(functionIndices, IsSelfInstallingImage ? TableBaseOffsetExpr : null); } } } diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs index 26409c341f5acb..2663e7048cb30b 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCases/R2RTestSuites.cs @@ -137,6 +137,7 @@ static void Validate(ReadyToRunReader reader) method.SignatureString.Contains("CatchException", StringComparison.Ordinal))); Assert.True(WasmR2RAssert.WasmIndexSpacesHaveExpectedEntries(webcilReader, out string indexDiagnostic), indexDiagnostic); + Assert.True(WasmR2RAssert.WasmSelfInstallingSegmentsHaveExpectedModes(webcilReader, out string segmentDiagnostic), segmentDiagnostic); // The wasm JIT references the ABI well-known globals via maximally padded WASM_GLOBAL_INDEX_LEB // relocations that the R2R object writer must self-resolve to the fixed global @@ -2042,9 +2043,12 @@ static void Validate(ReadyToRunReader reader) // Webcil segment 0 // | (byte) segment kind (1, passive) // | (ULEB) segment size - // | (byte*) content - 2 little endian u32 (payloadsize, tablesize) + // | (byte*) content - 2 little endian u32 (payloadsize, tablesize), then alignment padding // Webcil payload - // | (segment kind) (1, passive) + // | (segment kind) (0, active) + // | (byte) global.get + // | (ULEB) image base global index + // | (byte) end // | (ULEB) segment size // | (byte*) content - webcil data, aligned @@ -2055,12 +2059,29 @@ static void Validate(ReadyToRunReader reader) int firstSegmentKind = imageSpan[firstSegmentOffset]; Assert.True(firstSegmentKind == 1, "Expected first segment to be passive (kind 1)"); int firstSegmentSize = (int)DwarfHelper.ReadULEB128(imageSpan.Slice(firstSegmentOffset + 1), out int firstSegmentSizeBytes); + int firstSegmentContentOffset = firstSegmentOffset + 1 + firstSegmentSizeBytes; + const int SizeMetadataLength = sizeof(uint) * 2; + Assert.True(firstSegmentSize >= SizeMetadataLength, "Expected payload and table sizes in the first segment"); + foreach (byte padding in imageSpan.Slice(firstSegmentContentOffset + SizeMetadataLength, firstSegmentSize - SizeMetadataLength)) + { + Assert.Equal(0, padding); + } int payloadSegmentOffset = firstSegmentOffset + 1 + firstSegmentSizeBytes + firstSegmentSize; int payloadSegmentKind = imageSpan[payloadSegmentOffset]; - Assert.True(payloadSegmentKind == 1, "Expected second segment to be passive (kind 1)"); - int payloadSegmentSize = (int)DwarfHelper.ReadULEB128(imageSpan.Slice(payloadSegmentOffset + 1), out int payloadSegmentSizeBytes); - int payloadContentOffset = payloadSegmentOffset + 1 + payloadSegmentSizeBytes; + Assert.True(payloadSegmentKind == 0, "Expected second segment to be active (kind 0)"); + int payloadSizeOffset = payloadSegmentOffset + 1; + const byte GlobalGetOpcode = 0x23; + const byte EndOpcode = 0x0B; + Assert.Equal(GlobalGetOpcode, imageSpan[payloadSizeOffset++]); + int imageBaseGlobalIndex = (int)DwarfHelper.ReadULEB128(imageSpan.Slice(payloadSizeOffset), out int globalIndexBytes); + Assert.Equal(WebCilObjectWriter.ImageBaseGlobalIndex, imageBaseGlobalIndex); + payloadSizeOffset += globalIndexBytes; + Assert.Equal(EndOpcode, imageSpan[payloadSizeOffset++]); + int payloadSegmentSize = (int)DwarfHelper.ReadULEB128(imageSpan.Slice(payloadSizeOffset), out int payloadSegmentSizeBytes); + int payloadContentOffset = payloadSizeOffset + payloadSegmentSizeBytes; + Assert.Equal((uint)payloadSegmentSize, + System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(imageSpan.Slice(firstSegmentContentOffset, sizeof(uint)))); Assert.True(payloadContentOffset % WebCilObjectWriter.WebcilSectionAlignment == 0, $"Expected payload content to be aligned to {WebCilObjectWriter.WebcilSectionAlignment} bytes, but got offset {payloadContentOffset}"); Assert.True(payloadContentOffset + payloadSegmentSize == offset + sectionSize + 1 + sectionSizeBytes, diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCasesRunner/WasmR2RAssert.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCasesRunner/WasmR2RAssert.cs index 5f913584862f72..d7665d248f32b3 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCasesRunner/WasmR2RAssert.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/TestCasesRunner/WasmR2RAssert.cs @@ -65,13 +65,13 @@ public static bool WasmIndexSpacesHaveExpectedEntries(WebcilImageReader reader, Dictionary<(string Module, string Name), WasmImportIndex> imports = ReadWasmImports(reader); (string Name, WasmImportKind Kind, uint Index)[] expectedImports = [ - ("stackPointer", WasmImportKind.Global, 0), - ("imageBase", WasmImportKind.Global, 1), - ("tableBase", WasmImportKind.Global, 2), - ("asyncContinuation", WasmImportKind.Global, 3), - ("table", WasmImportKind.Table, 0), + ("__stack_pointer", WasmImportKind.Global, 0), + ("__memory_base", WasmImportKind.Global, 1), + ("__table_base", WasmImportKind.Global, 2), + ("__async_continuation", WasmImportKind.Global, 3), + ("__indirect_function_table", WasmImportKind.Table, 0), ("memory", WasmImportKind.Memory, 0), - ("rtlRestoreContextTag", WasmImportKind.Tag, 0), + ("__coreclr_wasm_rtlrestorecontext_tag", WasmImportKind.Tag, 0), ]; var failures = new List(); @@ -125,6 +125,102 @@ public static bool WasmIndexSpacesHaveExpectedEntries(WebcilImageReader reader, return failures.Count == 0; } + /// + /// Verifies that a code-carrying Webcil module installs its payload and function table through + /// active segments at the imported image and table base globals. + /// + public static bool WasmSelfInstallingSegmentsHaveExpectedModes(WebcilImageReader reader, out string diagnostic) + { + ReadOnlySpan image = reader.GetEntireImage().AsSpan(); + var failures = new List(); + + if (!TryGetWasmSectionBounds(image, WasmSectionKind.Data, out int dataOffset, out int dataEnd)) + { + failures.Add("WASM image does not contain a data section."); + } + else + { + uint dataSegmentCount = ReadWasmUleb32(image, ref dataOffset, dataEnd); + if (dataSegmentCount != 2) + { + failures.Add($"WASM data section contains {dataSegmentCount} segments; expected 2."); + } + else + { + uint countSegmentMode = ReadWasmUleb32(image, ref dataOffset, dataEnd); + if (countSegmentMode != 1) + failures.Add($"The webcilCount data segment has mode {countSegmentMode}; expected passive mode 1."); + + uint countSegmentSize = ReadWasmUleb32(image, ref dataOffset, dataEnd); + if (countSegmentSize > dataEnd - dataOffset) + throw new BadImageFormatException("The webcilCount data segment extends beyond the data section."); + dataOffset += (int)countSegmentSize; + + uint payloadSegmentMode = ReadWasmUleb32(image, ref dataOffset, dataEnd); + if (payloadSegmentMode != 0) + { + failures.Add($"The webcilPayload data segment has mode {payloadSegmentMode}; expected active mode 0."); + } + else + { + CheckGlobalGetOffsetExpression(image, ref dataOffset, dataEnd, expectedGlobalIndex: 1, "webcilPayload", failures); + } + } + } + + if (!TryGetWasmSectionBounds(image, WasmSectionKind.Element, out int elementOffset, out int elementEnd)) + { + failures.Add("WASM image does not contain an element section."); + } + else + { + uint elementSegmentCount = ReadWasmUleb32(image, ref elementOffset, elementEnd); + if (elementSegmentCount != 1) + { + failures.Add($"WASM element section contains {elementSegmentCount} segments; expected 1."); + } + else + { + uint elementSegmentMode = ReadWasmUleb32(image, ref elementOffset, elementEnd); + if (elementSegmentMode != 0) + { + failures.Add($"The function-table element segment has mode {elementSegmentMode}; expected active mode 0."); + } + else + { + CheckGlobalGetOffsetExpression(image, ref elementOffset, elementEnd, expectedGlobalIndex: 2, "function-table", failures); + } + } + } + + diagnostic = failures.Count == 0 + ? "WASM payload and function-table segments use the expected self-installing modes." + : string.Join(Environment.NewLine, failures); + return failures.Count == 0; + } + + private static void CheckGlobalGetOffsetExpression( + ReadOnlySpan image, + ref int offset, + int end, + uint expectedGlobalIndex, + string segmentName, + List failures) + { + const byte GlobalGetOpcode = 0x23; + const byte EndOpcode = 0x0B; + + byte opcode = ReadWasmByte(image, ref offset, end); + uint globalIndex = ReadWasmUleb32(image, ref offset, end); + byte endOpcode = ReadWasmByte(image, ref offset, end); + if (opcode != GlobalGetOpcode || globalIndex != expectedGlobalIndex || endOpcode != EndOpcode) + { + failures.Add( + $"The {segmentName} segment offset was opcode 0x{opcode:X2}, global {globalIndex}, " + + $"end 0x{endOpcode:X2}; expected global.get {expectedGlobalIndex}, end."); + } + } + private static uint CountWasmImports( Dictionary<(string Module, string Name), WasmImportIndex> imports, WasmImportKind kind) @@ -578,6 +674,7 @@ private enum WasmSectionKind : byte Global = 6, Export = 7, Element = 9, + Data = 11, Tag = 13, } diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/WebcilImageReader.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/WebcilImageReader.cs index 70f1f4757425d3..6217cea781c940 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/WebcilImageReader.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/WebcilImageReader.cs @@ -739,7 +739,9 @@ private static bool TryFindWebcilInWasm(byte[] image, out long webcilOffset) webcilOffset = 0; // Parse WASM module structure to find the data section (id=11) - // which contains the Webcil payload as a passive data segment. + // which contains the Webcil payload. The payload segment is passive in a component + // forwarding stub and active in a self-installing image (one that carries code and + // installs itself at instantiation), so both kinds have to be inspected. int offset = 8; // Skip WASM magic + version while (offset < image.Length) { @@ -756,45 +758,39 @@ private static bool TryFindWebcilInWasm(byte[] image, out long webcilOffset) if (sectionId == 11) // Data section { // Data section contains: count(LEB128) then count segments. - // Each passive segment: kind=1(byte) + size(LEB128) + bytes - // The Webcil payload is in the second passive data segment. + // Passive segment: kind=1(byte) + size(LEB128) + bytes + // Active segment: kind=0(byte) [+ memidx if kind=2] + offset expr + size + bytes uint segmentCount = ReadLebU32(image, ref offset); for (uint i = 0; i < segmentCount && offset < sectionEnd; i++) { byte kind = image[offset++]; - if (kind == 1) // Passive segment + if (kind == 2) // Active segment with an explicit memory index { - uint dataSize = ReadLebU32(image, ref offset); - // Check if this segment starts with the Webcil magic - if (dataSize >= 4 && offset + dataSize <= image.Length) - { - uint magic = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(offset)); - if (magic == WebcilConstants.WEBCIL_MAGIC && TryReadHeader(image, offset, out _)) - { - webcilOffset = offset; - return true; - } - } - offset += (int)dataSize; + ReadLebU32(image, ref offset); // memory index } - else if (kind == 0) // Active segment (memory 0) + else if (kind is not (0 or 1)) { - // Skip the init expression + data - SkipConstExpr(image, ref offset); - uint dataSize = ReadLebU32(image, ref offset); - offset += (int)dataSize; + return false; // Unknown segment kind } - else if (kind == 2) // Active segment (explicit memory index) + + if (kind != 1) // Active segments carry an offset constant expression { - ReadLebU32(image, ref offset); // memory index SkipConstExpr(image, ref offset); - uint dataSize = ReadLebU32(image, ref offset); - offset += (int)dataSize; } - else + + uint dataSize = ReadLebU32(image, ref offset); + // Check if this segment starts with the Webcil magic + if (dataSize >= 4 && offset + dataSize <= image.Length) { - return false; // Unknown segment kind + uint magic = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(offset)); + if (magic == WebcilConstants.WEBCIL_MAGIC && TryReadHeader(image, offset, out _)) + { + webcilOffset = offset; + return true; + } } + + offset += (int)dataSize; } return false; } diff --git a/src/mono/wasi/build/WasiApp.CoreCLR.targets b/src/mono/wasi/build/WasiApp.CoreCLR.targets index a1dd6dd842ceeb..7937590246d00d 100644 --- a/src/mono/wasi/build/WasiApp.CoreCLR.targets +++ b/src/mono/wasi/build/WasiApp.CoreCLR.targets @@ -32,6 +32,100 @@ false + + wasm + true + $(PublishReadyToRunCrossgen2ExtraArgs);--opt-cross-module:*;--codegenopt:JitWasmNyiToR2RUnsupported=1;--codegenopt:JitWasmSimdNyiToR2RUnsupported=1 + true + python + python3 + + + + + + + + + + + + $(MicrosoftNetCoreAppRuntimePackRidNativeDir) + $(MicrosoftNetCoreAppRuntimePackRidNativeDir) + $(MicrosoftNetCoreAppRuntimePackRidLibTfmDir)System.Private.CoreLib.dll + $(MicrosoftNetCoreAppRuntimePackRidNativeDir)System.Private.CoreLib.dll + + + + + + + + + + + <_WasiReadyToRunFrameworkInput Include="$(WasiCoreCLRSystemPrivateCoreLibPath)" /> + <_WasiReadyToRunFrameworkInput Include="$(MicrosoftNetCoreAppRuntimePackRidLibTfmDir)*.dll" + Exclude="$(WasiCoreCLRSystemPrivateCoreLibPath)" /> + + + + + + + <_WasiR2ROutputDir>$([MSBuild]::NormalizeDirectory('$(_ReadyToRunOutputPath)')) + <_WasiR2RCompositePath>$(_WasiR2ROutputDir)composite-r2r.wasm + <_WasiR2RImageBufferSource>$([MSBuild]::NormalizePath('$(_WasmIntermediateOutputPath)', 'corerun-relink', 'r2r-image-buffer.cpp')) + + + + + + + + + <_WasiR2RFunctionCount>$([System.String]::Copy('$(_WasiR2RDescription)').Split(',').GetValue(0)) + <_WasiR2RPayloadSize>$([System.String]::Copy('$(_WasiR2RDescription)').Split(',').GetValue(1)) + + $([MSBuild]::Add($(_WasiR2RFunctionCount), 1)) + + + + + + + + @@ -57,15 +151,17 @@ (linked below), which finds them via CORE_ROOT. See https://github.com/dotnet/runtime/issues/130129. --> - + - <_WasiCoreCLRCoreLib Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)System.Private.CoreLib.dll" /> + <_WasiCoreCLRCoreLib Include="$(WasiCoreCLRSystemPrivateCoreLibPath)" /> - <_WasiCoreCLRFrameworkFiles Include="$(MicrosoftNetCoreAppRuntimePackRidLibTfmDir)\*.dll" /> + <_WasiCoreCLRFrameworkFiles Include="$(MicrosoftNetCoreAppRuntimePackRidLibTfmDir)\*.dll" + Exclude="$(WasiCoreCLRSystemPrivateCoreLibPath)" /> <_WasiCoreCLRFrameworkToCopy Include="@(_WasiCoreCLRFrameworkFiles)" Condition="!Exists('$(WasmAppDir)managed\%(_WasiCoreCLRFrameworkFiles.FileName)%(_WasiCoreCLRFrameworkFiles.Extension)')" /> @@ -88,7 +184,7 @@ <_WasiCallHelperSource Include="$(_WasiReversePInvokeTablePath)" ObjectFile="$(_WasiRelinkObjDir)callhelpers-reverse.o" /> <_WasiCallHelperSource Include="$(_WasiInterpToNativeTablePath)" ObjectFile="$(_WasiRelinkObjDir)callhelpers-interp-to-managed.o" /> + <_WasiCallHelperSource Include="$(_WasiR2RImageBufferSource)" + ObjectFile="$(_WasiRelinkObjDir)r2r-image-buffer.o" + Condition="'$(PublishReadyToRun)' == 'true'" /> - <_WasiHostLibsPre Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libminipal.a" /> - <_WasiHostLibsPre Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libcoreclr_static.a" /> + <_WasiHostLibsPre Include="$(WasiCoreCLRNativeRuntimeDir)libminipal.a" /> + <_WasiHostLibsPre Include="$(WasiCoreCLRNativeRuntimeDir)libcoreclr_static.a" /> <_WasiHostLibsPre Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libSystem.IO.Compression.Native.a" /> <_WasiHostLibsPre Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libSystem.Native.a" /> <_WasiHostLibsPre Condition="'$(InvariantTimezone)' == 'true'" Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libSystem.Native.TimeZoneData.Invariant.a" /> <_WasiHostLibsPre Condition="'$(InvariantTimezone)' != 'true'" Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libSystem.Native.TimeZoneData.a" /> <_WasiHostLibsPre Include="%(_WasiCallHelperSource.ObjectFile)" /> - <_WasiHostLibsPre Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libnativeresourcestring.a" /> + <_WasiHostLibsPre Include="$(WasiCoreCLRNativeRuntimeDir)libnativeresourcestring.a" /> - <_WasiHostLibsPost Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libcoreclrminipal.a" /> - <_WasiHostLibsPost Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libcoreclrpal.a" /> - <_WasiHostLibsPost Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libminipal.a" /> + <_WasiHostLibsPost Include="$(WasiCoreCLRNativeRuntimeDir)libcoreclrminipal.a" /> + <_WasiHostLibsPost Include="$(WasiCoreCLRNativeRuntimeDir)libcoreclrpal.a" /> + <_WasiHostLibsPost Include="$(WasiCoreCLRNativeRuntimeDir)libminipal.a" /> <_WasiHostLibsPost Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libz.a" /> <_WasiHostLibsPost Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libzstd.a" Condition="Exists('$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libzstd.a')" /> - <_WasiRelinkOutput>$(WasmAppDir)managed\corerun + <_WasiRelinkOutput>$(_WasiRelinkObjDir)corerun + <_WasiPublishedHost>$(WasmAppDir)managed\corerun + + false + + 65537 @@ -308,6 +417,17 @@ <_WasiRelinkLinkFlags Include="$(_WasiRelinkOptFlag)" /> <_WasiRelinkLinkFlags Include="-DNDEBUG" Condition="'$(Configuration)' != 'Debug'" /> <_WasiRelinkLinkFlags Include="-Wl,--gc-sections" /> + + <_WasiRelinkLinkFlags Include="-Wl,--export=__async_continuation" /> + + <_WasiRelinkLinkFlags Include="-Wl,--table-base=$(WasiCompositeR2RTableBase)" Condition="'$(WasiEnableCompositeR2R)' == 'true'" /> + <_WasiRelinkLinkFlags Include="-Wl,--export-table" Condition="'$(WasiEnableCompositeR2R)' == 'true'" /> + <_WasiRelinkLinkFlags Include="-Wl,--export=__stack_pointer" Condition="'$(WasiEnableCompositeR2R)' == 'true'" /> + <_WasiRelinkLinkFlags Include="-Wl,--export=__coreclr_wasm_rtlrestorecontext_tag" Condition="'$(WasiEnableCompositeR2R)' == 'true'" /> <_WasiRelinkLinkFlags Include="-fwasm-exceptions" /> <_WasiRelinkLinkFlags Include="-mllvm" /> <_WasiRelinkLinkFlags Include="-wasm-use-legacy-eh=false" /> @@ -322,7 +442,7 @@ <_WasiRelinkLinkFlags Include="-Wl,--component-type,"$(_WasiHttpWorldWit.Replace('\','/'))"" /> <_WasiRelinkLinkFlags Include="-Wl,--whole-archive" /> - <_WasiRelinkLinkFlags Include=""$(MicrosoftNetCoreAppRuntimePackRidNativeDir)libWasiHost.a"" /> + <_WasiRelinkLinkFlags Include=""$(WasiCoreCLRHostNativeDir)libWasiHost.a"" /> <_WasiRelinkLinkFlags Include="-Wl,--no-whole-archive" /> <_WasiRelinkLinkFlags Include="@(_WasiHostLibsPre->'"%(Identity)"')" /> <_WasiRelinkLinkFlags Include="-lstdc++" /> @@ -334,10 +454,63 @@ + + + + + <_WasiR2RComposeOutputDir>$([MSBuild]::NormalizeDirectory('$(_WasmIntermediateOutputPath)', 'r2r-compose')) + <_WasiR2RComposedHost>$(_WasiR2RComposeOutputDir)corerun-composite.wasm + <_WasiR2RStubDir>$([MSBuild]::NormalizeDirectory('$(WasmAppDir)', 'managed', 'comp')) + + + + + + + <_WasiR2RComponentStub Include="@(_ReadyToRunCompositeBuildInput->'$(_WasiR2ROutputDir)%(FileName).wasm')" /> + <_WasiR2RComponentStub Include="@(_ReadyToRunCompositeUnrootedBuildInput->'$(_WasiR2ROutputDir)%(FileName).wasm')" /> + + + + + + + + + + + + $(WasmAppDir)run-wasmtime.sh + <_ScriptContent Condition="'$(WasmSingleFileBundle)' == 'true'">wasmtime run $([System.IO.Path]::GetFileNameWithoutExtension($(WasmMainAssemblyFileName))).wasm $* + <_ScriptContent Condition="'$(WasmSingleFileBundle)' != 'true'">cd managed && wasmtime run -W exceptions=y -S http --dir . corerun $(WasmMainAssemblyFileName) $* + + + + + + + + + diff --git a/src/mono/wasi/build/WasiApp.InTree.props b/src/mono/wasi/build/WasiApp.InTree.props index ae0f77312bfe23..faf5cd0d1603d9 100644 --- a/src/mono/wasi/build/WasiApp.InTree.props +++ b/src/mono/wasi/build/WasiApp.InTree.props @@ -8,11 +8,21 @@ WasiApp.CoreCLR.targets). --> false + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'wasi-wasm.$(Configuration)', 'sharedFramework')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', 'wasi.wasm.$(Configuration)', 'sharedFramework')) + $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'coreclr', 'wasi.wasm.$(Configuration)', 'IL', 'System.Private.CoreLib.dll')) + $(MSBuildThisFileDirectory)compose-r2r.py false + + $(AfterMicrosoftNETSdkTargets);$(Crossgen2SdkOverrideTargetsPath) + $(AfterMicrosoftNETSdkTargets);$(MSBuildThisFileDirectory)WasiApp.ReadyToRun.targets + true diff --git a/src/mono/wasi/build/WasiApp.ReadyToRun.targets b/src/mono/wasi/build/WasiApp.ReadyToRun.targets new file mode 100644 index 00000000000000..55fd76a0ad77a0 --- /dev/null +++ b/src/mono/wasi/build/WasiApp.ReadyToRun.targets @@ -0,0 +1,20 @@ + + + + + $([MSBuild]::NormalizePath('$(Crossgen2InBuildDir)', 'crossgen2$(ExeSuffix)')) + + + + + + + + diff --git a/src/mono/wasi/build/compose-r2r.py b/src/mono/wasi/build/compose-r2r.py new file mode 100644 index 00000000000000..22726b8dbc4494 --- /dev/null +++ b/src/mono/wasi/build/compose-r2r.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. + +"""Compose a wasm R2R image into a WASI host component. + +The composite crossgen2 emits is SELF-INSTALLING: the webcil payload is an ACTIVE data +segment at (global.get __memory_base) and the R2R function table is an ACTIVE element +segment at (global.get __table_base). The engine installs both at instantiation. + +corerun supplies five of the composite's seven imports directly (memory, __stack_pointer, +__indirect_function_table, __coreclr_wasm_rtlrestorecontext_tag, __async_continuation). +The remaining two are the base globals, which wasm-ld only creates in PIC mode -- so a +generated shim module supplies them instead. + +Requires: wasm-tools and binaryen (wasm-merge, wasm-opt). The shim is assembled here +and the host's reservation parameters are read directly from its module. + + COMP= CORERUN= OUTDIR= python3 compose-r2r.py +""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# ---------------------------------------------------------------- wasm reading + +SEC_CUSTOM, SEC_TYPE, SEC_IMPORT, SEC_FUNCTION = 0, 1, 2, 3 +SEC_GLOBAL, SEC_EXPORT, SEC_START = 6, 7, 8 +SEC_ELEMENT, SEC_CODE, SEC_DATA = 9, 10, 11 + +EXTERNKIND_FUNC = 0 + + +class WasmError(Exception): + pass + + +def _uleb(data, pos): + result = shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + result |= (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + return result, pos + if shift >= 64: + raise WasmError("invalid overlong ULEB128 value") + raise WasmError("truncated ULEB128 value") + + +def _sleb(data, pos): + result = shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + result |= (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + if shift < 64 and byte & 0x40: + result -= 1 << shift + return result, pos + if shift >= 64: + raise WasmError("invalid overlong SLEB128 value") + raise WasmError("truncated SLEB128 value") + + +def _emit_uleb(value): + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _emit_sleb(value): + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + done = (value == 0 and not byte & 0x40) or (value == -1 and byte & 0x40) + out.append(byte if done else byte | 0x80) + if done: + return bytes(out) + + +class WasmModule: + """Minimal core-module reader: just enough to answer the splice's questions.""" + + def __init__(self, path): + self.path = Path(path) + self.data = self.path.read_bytes() + if len(self.data) < 8 or self.data[:4] != b"\0asm": + raise WasmError(f"{path} is not a wasm core module (bad magic). " + "If it is a component, unbundle it first.") + self.sections = [] # (id, payload_start, payload_end) + pos = 8 + while pos < len(self.data): + sec_id = self.data[pos] + size, pos = _uleb(self.data, pos + 1) + if size > len(self.data) - pos: + raise WasmError(f"section {sec_id} in {self.path.name} extends beyond the file") + self.sections.append((sec_id, pos, pos + size)) + pos += size + + def _section(self, sec_id): + for sid, start, end in self.sections: + if sid == sec_id: + return start, end + return None + + def _vec(self, sec_id): + span = self._section(sec_id) + if span is None: + return 0, None + count, pos = _uleb(self.data, span[0]) + return count, pos + + def func_import_count(self): + count, pos = self._vec(SEC_IMPORT) + if pos is None: + return 0 + total = 0 + for _ in range(count): + for _ in range(2): # module, name + length, pos = _uleb(self.data, pos) + pos += length + kind = self.data[pos] + pos += 1 + if kind == EXTERNKIND_FUNC: + total += 1 + _, pos = _uleb(self.data, pos) + elif kind == 1: # table + pos += 1 + limits = self.data[pos] + pos += 1 + _, pos = _uleb(self.data, pos) + if limits: + _, pos = _uleb(self.data, pos) + elif kind == 2: # memory + limits = self.data[pos] + pos += 1 + _, pos = _uleb(self.data, pos) + if limits: + _, pos = _uleb(self.data, pos) + elif kind == 3: # global + pos += 2 + elif kind == 4: # tag + pos += 1 + _, pos = _uleb(self.data, pos) + else: + raise WasmError(f"unknown import kind {kind} in {self.path.name}") + return total + + def defined_func_count(self): + count, _ = self._vec(SEC_FUNCTION) + return count + + def exports(self): + """name -> (kind, index)""" + count, pos = self._vec(SEC_EXPORT) + found = {} + if pos is None: + return found + for _ in range(count): + length, pos = _uleb(self.data, pos) + name = self.data[pos:pos + length].decode("utf-8", "replace") + pos += length + kind = self.data[pos] + pos += 1 + index, pos = _uleb(self.data, pos) + found[name] = (kind, index) + return found + + def _code_body(self, defined_index): + count, pos = self._vec(SEC_CODE) + if pos is None or defined_index >= count: + return None + for i in range(count): + size, body = _uleb(self.data, pos) + if i == defined_index: + return self.data[body:body + size] + pos = body + size + return None + + def const_i32_export(self, name): + """Value of an exported function whose whole body is `i32.const N`. + + The host publishes each splice parameter this way so it decodes statically, with no + instantiation and no copy of the value living in this script. + """ + entry = self.exports().get(name) + if entry is None or entry[0] != EXTERNKIND_FUNC: + return None + defined = entry[1] - self.func_import_count() + if defined < 0: + return None # an imported function has no body to read + body = self._code_body(defined) + if not body: + return None + local_decls, pos = _uleb(body, 0) + for _ in range(local_decls): + _, pos = _uleb(body, pos) + pos += 1 + if body[pos] != 0x41: # i32.const + return None + value, pos = _sleb(body, pos + 1) + return value if body[pos] == 0x0B else None + + def _const_offset(self, pos): + """Decode a constant init_expr. Returns (value_or_None, kind, next_pos).""" + op = self.data[pos] + if op == 0x41: # i32.const + value, pos = _sleb(self.data, pos + 1) + return value, "i32.const", pos + 1 # skip 0x0B + if op == 0x23: # global.get + index, pos = _uleb(self.data, pos + 1) + return index, "global.get", pos + 1 + raise WasmError(f"unsupported init_expr opcode 0x{op:02x} in {self.path.name}") + + def element_segments(self): + count, pos = self._vec(SEC_ELEMENT) + out = [] + if pos is None: + return out + for _ in range(count): + flags, pos = _uleb(self.data, pos) + seg = {"flags": flags, "active": flags in (0, 2, 4, 6), "offset": None, + "offset_kind": None, "count": 0} + if flags in (2, 6): + _, pos = _uleb(self.data, pos) # table index + if seg["active"]: + value, kind, pos = self._const_offset(pos) + seg["offset"], seg["offset_kind"] = value, kind + if flags in (1, 2, 5, 6): + pos += 1 # elemkind / reftype + elif flags in (3, 7): + pos += 1 + n, pos = _uleb(self.data, pos) + seg["count"] = n + for _ in range(n): + _, pos = _uleb(self.data, pos) + out.append(seg) + return out + + def data_segments(self): + count, pos = self._vec(SEC_DATA) + out = [] + if pos is None: + return out + for _ in range(count): + flags, pos = _uleb(self.data, pos) + seg = {"flags": flags, "active": flags in (0, 2), "offset": None, + "offset_kind": None, "size": 0} + if flags == 2: + _, pos = _uleb(self.data, pos) # memory index + if seg["active"]: + value, kind, pos = self._const_offset(pos) + seg["offset"], seg["offset_kind"] = value, kind + size, pos = _uleb(self.data, pos) + seg["size"] = size + pos += size + out.append(seg) + return out + + +def make_shim(memory_base, table_base, patch_header, payload_size): + """Assemble the module supplying what wasm-ld cannot emit for a non-PIC main module. + + Always exports the two base globals. When `patch_header` is set it additionally imports + the composite's `patchWebcilHeader` and calls it from a start function, so the composite + fills in its own header's TableBase field using the `__table_base` this shim defines. + That is strictly better than the host writing that field: the composite owns both the + offset and the value, so the two cannot disagree about the format. + + The import lives here rather than in corerun deliberately. corerun is a WASI *component*, + and an arbitrary core import is not expressible in a WIT world -- `wasm-component-ld` + rejects it with "failed to decode world from module". The shim is merged and never + componentized, so the import is resolved by the merge and the component wrapper never + sees it. + + Hand-assembled rather than written as WAT so wabt is not a prerequisite; the caller + validates the result before it reaches the merge. + """ + def section(sec_id, payload): + return bytes([sec_id]) + _emit_uleb(len(payload)) + payload + + def name(text): + raw = text.encode("utf-8") + return _emit_uleb(len(raw)) + raw + + out = bytearray(b"\0asm\x01\x00\x00\x00") + + if patch_header: + # (i32,i32)->() for patchWebcilHeader, and ()->() for the start function. + types = _emit_uleb(2) + b"\x60\x02\x7f\x7f\x00" + b"\x60\x00\x00" + out += section(SEC_TYPE, types) + out += section(SEC_IMPORT, + _emit_uleb(1) + name("composite") + name("patchWebcilHeader") + + b"\x00" + _emit_uleb(0)) + out += section(SEC_FUNCTION, _emit_uleb(1) + _emit_uleb(1)) + + def global_entry(value): + return b"\x7f\x00\x41" + _emit_sleb(value) + b"\x0b" # i32, immutable, i32.const + + out += section(SEC_GLOBAL, + _emit_uleb(2) + global_entry(memory_base) + global_entry(table_base)) + out += section(SEC_EXPORT, + _emit_uleb(2) + + name("__memory_base") + b"\x03" + _emit_uleb(0) + + name("__table_base") + b"\x03" + _emit_uleb(1)) + + if patch_header: + # Function 0 is the import, so the start function is index 1. + out += section(SEC_START, _emit_uleb(1)) + # Let the composite validate the complete payload extent rather than duplicating its + # current header size in the host-side shim. + body = (_emit_uleb(0) # no locals + + b"\x41" + _emit_sleb(memory_base) # i32.const dest + + b"\x41" + _emit_sleb(payload_size) # i32.const length + + b"\x10" + _emit_uleb(0) # call 0 + + b"\x0b") # end + out += section(SEC_CODE, _emit_uleb(1) + _emit_uleb(len(body)) + body) + + return bytes(out) + + +def swap_core_module(component_path, module_path, out_path): + """Replace the first core-module section of a component with the merged module.""" + component = Path(component_path).read_bytes() + merged = Path(module_path).read_bytes() + out = bytearray(component[:8]) + pos, swapped = 8, False + while pos < len(component): + sec_id = component[pos] + start = pos + size, pos = _uleb(component, pos + 1) + if sec_id == 1 and not swapped: # core module + out += bytes([1]) + _emit_uleb(len(merged)) + merged + swapped = True + else: + out += component[start:pos + size] + pos += size + if not swapped: + raise WasmError(f"{component_path} has no core-module section to replace") + Path(out_path).write_bytes(bytes(out)) + + +# ---------------------------------------------------------------- external tools + +def tool(name): + found = shutil.which(name) + if found is None: + raise WasmError(f"required tool '{name}' is not on PATH. " + "Install wasm-tools and binaryen; see docs/workflow/building/coreclr/wasi-r2r.md.") + return found + + +def run(args, capture=False): + result = subprocess.run(args, check=False, text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None) + if result.returncode != 0: + detail = f"\n{result.stdout.strip()}" if capture and result.stdout else "" + raise WasmError(f"{Path(args[0]).name} failed (exit {result.returncode}){detail}") + return result.stdout if capture else "" + + +# ---------------------------------------------------------------- pipeline + +def composite_requirements(composite): + n_funcs = composite.defined_func_count() + + # The payload is the composite's ONE active data segment. Select it by meaning: the + # 9-byte webcilCount segment is passive, so index-based selection would be a positional + # assumption that breaks silently if crossgen2 ever reorders segments. + payloads = [s for s in composite.data_segments() if s["active"]] + if len(payloads) != 1: + raise WasmError( + f"expected exactly one active data segment in {composite.path.name} " + f"(the webcil payload), found {len(payloads)}. The composite layout changed; " + "this check would otherwise pick the wrong segment.") + + return n_funcs, payloads[0]["size"] + + +def main(): + if len(sys.argv) == 3 and sys.argv[1] in ("--describe", "--function-count", "--payload-size"): + composite = WasmModule(sys.argv[2]) + n_funcs, payload = composite_requirements(composite) + if sys.argv[1] == "--describe": + print(f"{n_funcs},{payload}") + else: + print(n_funcs if sys.argv[1] == "--function-count" else payload) + return + + if len(sys.argv) != 1: + raise WasmError( + "usage: compose-r2r.py [--describe|--function-count|--payload-size] " + "") + + missing = [name for name in ("COMP", "CORERUN", "OUTDIR") if not os.environ.get(name)] + if missing: + raise WasmError(f"set the required environment variables: {', '.join(missing)}") + comp = Path(os.environ["COMP"]) + corerun = Path(os.environ["CORERUN"]) + outdir = Path(os.environ["OUTDIR"]) + + for label, path in (("composite", comp), ("corerun", corerun)): + if not path.is_file(): + raise WasmError(f"{label} not found at {path}") + outdir.mkdir(parents=True, exist_ok=True) + + wasm_tools = tool("wasm-tools") + + # 1. Unbundle the corerun component -> core module. Mandatory: corerun is a WASI + # component, and a component is not a core module. + run([wasm_tools, "component", "unbundle", str(corerun), + "--module-dir", str(outdir), "-o", os.devnull], capture=True) + modules = sorted(outdir.glob("*module0*.wasm")) + if not modules: + raise WasmError(f"unbundling {corerun.name} produced no *module0*.wasm in {outdir}") + host = WasmModule(modules[0]) + + # 2. Read the R2R parameters out of the LINKED host. The host owns these values; this + # script must not carry its own copy, or a rebuild with different settings silently + # produces a mismatched image. + params = {} + for key, export in (("image_base", "wasi_r2r_image_base"), + ("cap", "wasi_r2r_image_cap"), + ("table_base", "wasi_r2r_table_base")): + value = host.const_i32_export(export) + if value is None: + raise WasmError( + f"the host does not export {export} as an R2R parameter.\n" + " Link it with CORERUN_WASI_COMPOSITE_R2R=ON (corerun) or\n" + " WasiEnableCompositeR2R=true (apps); without those flags the probe\n" + " is present but can never be satisfied.") + params[key] = value + + composite = WasmModule(comp) + + # The composite installs at table_base and must end before the host's OWN element + # segment begins -- not merely inside the table. Both are ACTIVE in the merged module, + # so an overlap silently overwrites the host's function pointers rather than failing. + host_active = [s for s in host.element_segments() + if s["active"] and s["offset_kind"] == "i32.const"] + if not host_active: + raise WasmError("the host has no active element segment, so it reserves no table " + "slots for the composite.") + reserved = min(s["offset"] for s in host_active) + + n_funcs, payload = composite_requirements(composite) + + print(f"SHIM: imageBase={params['image_base']} tableBase={params['table_base']} " + f"reservedSlots={reserved} compositeFuncs={n_funcs} payload={payload} " + f"cap={params['cap']}") + + if params["table_base"] + n_funcs > reserved: + need = params["table_base"] + n_funcs + raise WasmError( + f"composite needs slots {params['table_base']}..{need - 1} but the host's own\n" + f" functions begin at {reserved}. They would overlap and silently corrupt\n" + f" dispatch. Raise the table base to at least {need}:\n" + f" corerun -DCORERUN_WASI_R2R_TABLE_BASE={need}\n" + f" apps -p:WasiCompositeR2RTableBase={need}") + + # The engine installs the payload into the host's staging buffer BEFORE any host code + # runs, so the host's own cap test cannot protect that buffer. This is the only place + # it is enforceable. + if payload > params["cap"]: + raise WasmError( + f"composite payload {payload} bytes exceeds the host's staging buffer " + f"({params['cap']}).\n" + " Raise WASI_R2R_IMAGE_CAP in corerun/wasi_r2r_probe.hpp and rebuild.") + + # 3. Generate the shim supplying the two globals wasm-ld cannot emit for a non-PIC main + # module, and validate it before it reaches the merge. + # + # If the composite exports patchWebcilHeader (the self-installing shape in + # docs/design/mono/webcil.md), have the shim call it from a start function so the + # composite writes its own TableBase field. Older composites predate that export, so + # detect rather than assume -- importing a function the composite does not export makes + # the merge leave it unresolved and the host fails to instantiate. When it is absent the + # host's own fallback in wasi_r2r_probe.hpp writes the field instead. + patch_header = "patchWebcilHeader" in composite.exports() + shim = outdir / "shim.wasm" + shim.write_bytes(make_shim( + params["image_base"], params["table_base"], patch_header, payload)) + run([wasm_tools, "validate", "--features", "all", str(shim)], capture=True) + print(f"SHIM: tableBase written by " + f"{'the composite (patchWebcilHeader)' if patch_header else 'the host (fallback)'}") + + # 4. Merge host + shim + composite. --enable-gc is needed only for the INTERMEDIATE: + # merging internalizes the imported globals, and global.get of a *defined* global is + # a constant expression only under the GC proposal. Step 5 removes that requirement. + # -g carries the name section through; see step 5 for why that matters. + merged = outdir / "merged.wasm" + run([tool("wasm-merge"), "-g", "--all-features", "--enable-gc", + str(modules[0]), "webcil", str(shim), "webcil", str(comp), "composite", + "-o", str(merged)], capture=True) + + # 5. Fold global.get -> i32.const so the result is MVP-valid; without this wasmtime + # rejects the module unless the embedder enables GC. Costs ~3.7% code size. + # -g preserves the name section. Since #132906 that section is the ONLY record of + # function names, and dropping it still validates and still runs -- the sole symptom + # is that every frame goes anonymous. + final = outdir / "final.wasm" + run([tool("wasm-opt"), str(merged), "--all-features", "-g", "--simplify-globals", + "-o", str(final)], capture=True) + + # 6. Swap the merged core module back into the corerun component. + out = outdir / "corerun-composite.wasm" + swap_core_module(corerun, final, out) + + run([wasm_tools, "validate", "--features", "all", str(out)], capture=True) + print("VALID") + print(f"OUT: {out}") + + +if __name__ == "__main__": + try: + main() + except WasmError as error: + print(f"error: {error}", file=sys.stderr) + sys.exit(1) + except IndexError: + print("error: malformed wasm input ended unexpectedly", file=sys.stderr) + sys.exit(1) diff --git a/src/native/corehost/wasihost/wasihost.cpp b/src/native/corehost/wasihost/wasihost.cpp index 9d34a920480a9d..eb91e160b604f3 100644 --- a/src/native/corehost/wasihost/wasihost.cpp +++ b/src/native/corehost/wasihost/wasihost.cpp @@ -17,6 +17,23 @@ // Shared pal (path handling, CORE_ROOT/TPA helpers); header-only, so no corerun object is linked. #include "corerun.hpp" +// Shared WASI R2R external-assembly probe (same code corerun uses), so the per-app test host serves +// statically-composed R2R images instead of silently interpreting everything. Requires corerun.hpp +// above (pal::try_map_file_readonly). +#define WASI_R2R_EXTERNAL_IMAGE_BUFFER +#include "wasi_r2r_probe.hpp" + +namespace wasi_r2r +{ +// A ReadyToRun publish supplies strong definitions sized from its composite. Keep non-R2R app links +// working without paying the 16 MiB development-host reservation. +extern "C" +{ + alignas(16) __attribute__((weak)) uint8_t g_wasi_r2r_image[64] = {}; + __attribute__((weak)) uint32_t g_wasi_r2r_image_cap = sizeof(g_wasi_r2r_image); +} +} + #include using pal::char_t; @@ -70,11 +87,36 @@ extern "C" __attribute__((weak)) int32_t GlobalizationNative_LoadICUData(const c static std::vector s_property_keys; static std::vector s_property_values; +// R2R external-assembly probe search dirs, captured before coreclr_initialize so the probe callback +// (invoked later by the runtime) can reach them. +static string_t s_r2r_core_root; +static string_t s_r2r_core_libs; + static void log_error_info(const char* line) { std::fprintf(stderr, "%s\n", line); } +// Serves statically-composed R2R images (the composite plus per-assembly stubs) to the runtime, using +// the shared WASI probe. Returns false for everything else, so non-R2R assemblies load normally via the +// TPA list. +static bool HOST_CONTRACT_CALLTYPE external_assembly_probe( + const char* path, + void** data_start, + int64_t* size) +{ + const char* name = path; + const char* slash = ::strrchr(name, '/'); + if (slash != nullptr) + name = slash + 1; + + const char* const r2r_dirs[] = { + s_r2r_core_libs.empty() ? nullptr : s_r2r_core_libs.c_str(), + s_r2r_core_root.empty() ? nullptr : s_r2r_core_root.c_str() + }; + return wasi_r2r::WasiStaticR2RProbe(name, r2r_dirs, 2, data_start, size); +} + // Include only the first instance of each simple assembly name (CoreCLR may otherwise prefer a // later ni over an earlier il). static string_t build_tpa(const string_t& core_root, const string_t& core_libraries) @@ -172,6 +214,10 @@ int main(int argc, char* argv[]) core_root = app_path; pal::ensure_trailing_delimiter(core_root); + // Capture the R2R probe search dirs (trailing-delimited) for the external_assembly_probe callback. + s_r2r_core_root = core_root; + s_r2r_core_libs = core_libs; + string_t exe_path = pal::get_exe_path(); string_t tpa_list = build_tpa(core_root, core_libs); @@ -193,6 +239,7 @@ int main(int argc, char* argv[]) static host_runtime_contract host_contract = { sizeof(host_runtime_contract), nullptr }; host_contract.get_runtime_property = &get_runtime_property; host_contract.pinvoke_override = &callhelpers_pinvoke_override; + host_contract.external_assembly_probe = &external_assembly_probe; { std::stringstream ss; ss << "0x" << std::hex << (size_t)(&host_contract); diff --git a/src/native/libs/Common/JavaScript/host/assets.ts b/src/native/libs/Common/JavaScript/host/assets.ts index e70315b5f6332c..ce7c1648b40a4f 100644 --- a/src/native/libs/Common/JavaScript/host/assets.ts +++ b/src/native/libs/Common/JavaScript/host/assets.ts @@ -105,10 +105,10 @@ function allocWebcilPayload(payloadSize: number): number { // Builds the `webcil` import object. For R2R images (tableSize > 0) the module imports the runtime's // stack pointer, exception tag, indirect-call table and base globals; this also grows the table. -// These import names and the webcilVersion/getWebcilPayload/fillWebcilTable handshake in -// finishWebcilInstance are the R2R Webcil-in-Wasm host ABI defined by crossgen's WasmObjectWriter -// (src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs, CreateDefaultGlobalImports/ -// WriteExports). Keep in sync with the corerun host +// These import names and the webcilVersion/getWebcilPayload/fillWebcilTable/patchWebcilHeader +// handshake in finishWebcilInstance are the R2R Webcil-in-Wasm host ABI defined by crossgen's +// WasmObjectWriter (src/coreclr/tools/Common/Compiler/ObjectWriter/WasmObjectWriter.cs, +// CreateDefaultGlobalImports/WriteExports). Keep in sync with the corerun host // (src/coreclr/hosts/corerun/wasm/libCorerun.js, BrowserHost_ExternalAssemblyProbe). Unlike corerun, // which parses data segment 0 for payloadSize/tableSize, this loader receives them from boot config. function buildWebcilImports(memory: WebAssembly.Memory, payloadPtr: number, tableSize: number): Record { @@ -128,12 +128,12 @@ function buildWebcilImports(memory: WebAssembly.Memory, payloadPtr: number, tabl } const tableStartIndex = _ems_.wasmTable.length; _ems_.wasmTable.grow(tableSize); - webcilImports.stackPointer = stackPointer; - webcilImports.rtlRestoreContextTag = rtlRestoreContextTag as unknown as WebAssembly.ImportValue; - webcilImports.asyncContinuation = asyncContinuation as unknown as WebAssembly.ImportValue; - webcilImports.table = _ems_.wasmTable; - webcilImports.tableBase = new WebAssembly.Global({ value: "i32", mutable: false }, tableStartIndex); - webcilImports.imageBase = new WebAssembly.Global({ value: "i32", mutable: false }, payloadPtr); + webcilImports.__stack_pointer = stackPointer; + webcilImports.__coreclr_wasm_rtlrestorecontext_tag = rtlRestoreContextTag as unknown as WebAssembly.ImportValue; + webcilImports.__async_continuation = asyncContinuation as unknown as WebAssembly.ImportValue; + webcilImports.__indirect_function_table = _ems_.wasmTable; + webcilImports.__table_base = new WebAssembly.Global({ value: "i32", mutable: false }, tableStartIndex); + webcilImports.__memory_base = new WebAssembly.Global({ value: "i32", mutable: false }, payloadPtr); } return webcilImports; } @@ -146,11 +146,21 @@ function finishWebcilInstance(instance: WebAssembly.Instance, payloadPtr: number throw new Error(`Unsupported Webcil version: ${webcilVersion}`); } - const getWebcilPayload = instance.exports.getWebcilPayload as (ptr: number, size: number) => void; - getWebcilPayload(payloadPtr, payloadSize); - if (tableSize > 0) { - const fillWebcilTable = instance.exports.fillWebcilTable as () => void; - fillWebcilTable(); + // Two image shapes reach this point. A component stub carries its payload and table in passive + // segments and hands them over via getWebcilPayload/fillWebcilTable. A composite uses active + // segments, so the engine installed both at instantiation and only the header's tableBase field + // is left to write. Feature-detect rather than assume: getWebcilPayload on a composite would + // trap, because memory.init against an active (hence dropped) segment is out of bounds. + const patchWebcilHeader = instance.exports.patchWebcilHeader as ((ptr: number, size: number) => void) | undefined; + if (typeof patchWebcilHeader === "function") { + patchWebcilHeader(payloadPtr, payloadSize); + } else { + const getWebcilPayload = instance.exports.getWebcilPayload as (ptr: number, size: number) => void; + getWebcilPayload(payloadPtr, payloadSize); + if (tableSize > 0) { + const fillWebcilTable = instance.exports.fillWebcilTable as () => void; + fillWebcilTable(); + } } const name = virtualPath.startsWith(browserVirtualAppBase) diff --git a/src/tasks/Crossgen2Tasks/Microsoft.NET.CrossGen.targets b/src/tasks/Crossgen2Tasks/Microsoft.NET.CrossGen.targets index 4ebf5ebb634321..e2a311cad2e8e0 100644 --- a/src/tasks/Crossgen2Tasks/Microsoft.NET.CrossGen.targets +++ b/src/tasks/Crossgen2Tasks/Microsoft.NET.CrossGen.targets @@ -385,9 +385,14 @@ Copyright (c) .NET Foundation. All rights reserved. and do not replace them. IL PDBs are still required for debugging. Native PDBs emitted by the R2R compiler are only used for profiling purposes. --> - - - + + + + diff --git a/src/tasks/Crossgen2Tasks/PrepareForReadyToRunCompilation.cs b/src/tasks/Crossgen2Tasks/PrepareForReadyToRunCompilation.cs index 09bdade3adcf4a..43ca1856a04ec3 100644 --- a/src/tasks/Crossgen2Tasks/PrepareForReadyToRunCompilation.cs +++ b/src/tasks/Crossgen2Tasks/PrepareForReadyToRunCompilation.cs @@ -346,6 +346,13 @@ private TaskItem CreateReadyToRunFileToPublish( out string compilerOutputRelativePath, out string compilerOutputPath) { + if (isCompositeImage && Crossgen2ContainerFormat == "wasm" && + Crossgen2Tool?.GetMetadata(MetadataKeys.TargetOS) == "wasi") + { + // The WASI composition pipeline consumes this fixed composite image name. + relativePath = "composite-r2r.wasm"; + } + // Crossgen2 emits WebAssembly directly, while Mach-O composite output is an object file // that must be linked into the dylib published by the SDK. (string compilerExtension, string publishExtension) = Crossgen2ContainerFormat switch diff --git a/src/tests/Common/CLRTest.CrossGen.targets b/src/tests/Common/CLRTest.CrossGen.targets index 4736289b4d401a..127f485b0b62a9 100644 --- a/src/tests/Common/CLRTest.CrossGen.targets +++ b/src/tests/Common/CLRTest.CrossGen.targets @@ -66,6 +66,10 @@ fi # CrossGen2 Script if [ ! -z ${RunCrossGen2+x} ]%3B then export TEST_READY_TO_RUN_MODE=1 + if [ "$(TargetOS)" == "wasi" ]; then + # WASI loads R2R code from a single composite spliced into the host. + export CompositeBuildMode=1 + fi compilationDoneFlagFile="IL-CG2/done" if [ -d IL-CG2 ]%3B then while [ ! -f $compilationDoneFlagFile ]%3B @@ -235,6 +239,21 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then done fi + if [ "$(TargetOS)" == "wasi" ] && [ $__cg2ExitCode -eq 0 ] && [ $__r2rDumpExitCode -eq 0 ]; then + ( + mkdir -p IL-CG2/wasi-r2r/comp || exit 1 + for dllFile in "$PWD"/IL-CG2/*.dll; do + bareFileName="${dllFile##*/}" + cp "$PWD/${bareFileName%.dll}.wasm" IL-CG2/wasi-r2r/comp/ || exit 1 + done + COMP="$PWD/composite-r2r.wasm" CORERUN="$CORE_ROOT/corerun" OUTDIR="$PWD/IL-CG2/wasi-r2r" python3 "$CORE_ROOT/wasi-r2r/compose-r2r.py" + ) + __linkExitCode=$? + if [ $__linkExitCode -ne 0 ]; then + rm -f "$PWD/IL-CG2/wasi-r2r/corerun-composite.wasm" + fi + fi + echo "Crossgen2 compilation finished, exit code $__cg2ExitCode" >> $compilationDoneFlagFile if [ $__cg2ExitCode -ne 0 ]; then echo Crossgen2 failed with exitcode: $__cg2ExitCode diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index b6e5b625928611..3f8473bb75fb68 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -272,7 +272,7 @@ fi block below and the matching block in corerun.cpp. wasmtime is expected to already be on PATH (provisioned via wasi-provisioning.targets or installed manually). --> - wasmtime run -W exceptions=y -S http --dir "$PWD::/" --dir "$CORE_ROOT::/core" --env CORE_ROOT=/core --env DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true --env DOTNET_WASI_PRINT_EXIT_CODE=1 "$CORE_ROOT/corerun" $(CoreRunArgs) ${__DotEnvArg} + wasmtime run -W exceptions=y -S http --dir "$PWD::/" --dir "$CORE_ROOT::/core" --env CORE_ROOT=/core --env DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true --env DOTNET_WASI_PRINT_EXIT_CODE=1 ${__WasiR2RArgs} "$__WasiCorerun" $(CoreRunArgs) ${__DotEnvArg} "$CORE_ROOT/watchdog" $_WatcherTimeoutMins + $([MSBuild]::NormalizeDirectory($(ArtifactsObjDir), 'helix-staging', 'wasmtime')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'helix-staging', 'wasi-r2r-tools', '$(BinaryenVersion)-$(WasmToolsVersion)-$(WasmToolHostRid)')) + + + <_BinaryenFilesToStage Include="$(BinaryenCacheDir)**\*" /> + <_WasmToolsFilesToStage Include="$(WasmToolsCacheDir)**\*" /> + + + + + + $(TestBinDir)Tests\Core_Root\ $([MSBuild]::NormalizeDirectory($(CoreRootDirectory))) @@ -705,6 +730,7 @@ + @@ -771,6 +797,7 @@ Uses the staged copy (see WasmtimeDirForHelixPayload) rather than $(WasmtimeDir) directly, matching sendtohelix-wasi.targets. --> + diff --git a/src/tests/Directory.Build.props b/src/tests/Directory.Build.props index 66f9d1fe232e6d..ed9f37c8dbecbd 100644 --- a/src/tests/Directory.Build.props +++ b/src/tests/Directory.Build.props @@ -206,6 +206,9 @@ /p:MSBuildEnableWorkloadResolver=false /p:Configuration=$(Configuration) + + + wasm