From 89d4f74f3a3c208944ad8525d1916511f2a12851 Mon Sep 17 00:00:00 2001 From: pavelsavara Date: Wed, 22 Feb 2023 18:43:41 +0100 Subject: [PATCH] wip --- eng/liveBuilds.targets | 1 - eng/native/configurecompiler.cmake | 5 +- .../Directory.Build.props | 2 +- .../Interop.GetTimeZoneData.Wasm.cs | 14 + .../System.Private.CoreLib.Shared.projitems | 3 + .../System/TimeZoneInfo.Unix.NonAndroid.cs | 144 +++++++--- src/mono/mono/utils/mono-dl-wasm.c | 43 +++ src/mono/mono/utils/mono-dl.h | 5 + src/mono/wasi/build/WasiApp.Native.targets | 44 ++- src/mono/wasi/build/WasiApp.targets | 1 - src/mono/wasi/build/WasiSdk.Defaults.props | 2 +- src/mono/wasi/runtime/CMakeLists.txt | 1 + src/mono/wasi/runtime/driver.c | 11 +- src/mono/wasi/runtime/main.c | 6 - src/mono/wasi/wasi.proj | 95 +++++-- .../wasm/Wasm.Build.Tests/BuildTestBase.cs | 1 - src/mono/wasm/build/WasmApp.targets | 1 - src/mono/wasm/runtime/CMakeLists.txt | 1 + src/mono/wasm/runtime/assets.ts | 5 - src/mono/wasm/runtime/driver.c | 3 + src/mono/wasm/runtime/startup.ts | 8 +- src/mono/wasm/wasm.proj | 99 +++++-- src/native/libs/System.Native/entrypoints.c | 1 + src/native/libs/System.Native/pal_datetime.c | 20 ++ src/native/libs/System.Native/pal_datetime.h | 2 + src/tasks/Common/Utils.cs | 5 +- .../WasmAppBuilder/EmitWasmBundleFiles.cs | 268 ++++++++++++++++++ src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 1 - .../wasi/EmitWasmBundleObjectFile.cs | 256 ----------------- .../wasi/WasmResolveAssemblyDependencies.cs | 199 ------------- .../WasmBuildTasks/GenerateWasmBundle.cs | 75 ----- 31 files changed, 670 insertions(+), 652 deletions(-) create mode 100644 src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs create mode 100644 src/tasks/WasmAppBuilder/EmitWasmBundleFiles.cs delete mode 100644 src/tasks/WasmAppBuilder/wasi/EmitWasmBundleObjectFile.cs delete mode 100644 src/tasks/WasmAppBuilder/wasi/WasmResolveAssemblyDependencies.cs delete mode 100644 src/tasks/WasmBuildTasks/GenerateWasmBundle.cs diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 5df409f12a259..5c12f4e63a3b7 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -192,7 +192,6 @@ $(LibrariesNativeArtifactsPath)package.json; $(LibrariesNativeArtifactsPath)dotnet.wasm; $(LibrariesNativeArtifactsPath)dotnet.js.symbols; - $(LibrariesNativeArtifactsPath)dotnet.timezones.blat; $(LibrariesNativeArtifactsPath)*.dat;" IsNative="true" /> diff --git a/eng/native/configurecompiler.cmake b/eng/native/configurecompiler.cmake index 0930fc03e274f..2b32e4d46b655 100644 --- a/eng/native/configurecompiler.cmake +++ b/eng/native/configurecompiler.cmake @@ -584,6 +584,9 @@ if(CLR_CMAKE_TARGET_UNIX) add_compile_definitions($<$>>:TARGET_ANDROID>) elseif(CLR_CMAKE_TARGET_LINUX) add_compile_definitions($<$>>:TARGET_LINUX>) + if(CLR_CMAKE_TARGET_BROWSER) + add_compile_definitions($<$>>:TARGET_BROWSER>) + endif() if(CLR_CMAKE_TARGET_LINUX_MUSL) add_compile_definitions($<$>>:TARGET_LINUX_MUSL>) endif() @@ -597,8 +600,6 @@ if(CLR_CMAKE_TARGET_UNIX) endif() elseif(CLR_CMAKE_TARGET_WASI) add_compile_definitions($<$>>:TARGET_WASI>) -elseif(CLR_CMAKE_TARGET_BROWSER) - add_compile_definitions($<$>>:TARGET_BROWSER>) else(CLR_CMAKE_TARGET_UNIX) add_compile_definitions($<$>>:TARGET_WINDOWS>) endif(CLR_CMAKE_TARGET_UNIX) diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props index 5f56e0eb0bbea..848f426f9667c 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props @@ -221,13 +221,13 @@ + - diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs new file mode 100644 index 0000000000000..1444fb0d551b8 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class Sys + { + [LibraryImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_GetTimeZoneData", StringMarshalling = StringMarshalling.Utf8, SetLastError = true)] + internal static partial IntPtr GetTimeZoneData(string fileName, out int length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 2a8355cc8399c..24c17a44bfd79 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -2143,6 +2143,9 @@ Common\Interop\Unix\System.Native\Interop.GetDefaultTimeZone.AnyMobile.cs + + Common\src\Interop\Unix\System.Native\Interop.GetTimeZoneData.Wasm.cs + Common\Interop\Unix\System.Native\Interop.GetEnv.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs b/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs index e9b915ed0fca1..f3a334959efe8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs +++ b/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs @@ -18,6 +18,11 @@ public sealed partial class TimeZoneInfo private const string TimeZoneDirectoryEnvironmentVariable = "TZDIR"; private const string TimeZoneEnvironmentVariable = "TZ"; + #if TARGET_WASI || TARGET_BROWSER + // if TZDIR is set, then the embedded TZ data will be ignored and normal unix behavior will be used + private static readonly bool UseEmbeddedTzDatabase = Environment.GetEnvironmentVariable(TimeZoneDirectoryEnvironmentVariable) == null; + #endif + private static TimeZoneInfo GetLocalTimeZoneCore() { // Without Registry support, create the TimeZoneInfo from a TZ file @@ -29,9 +34,30 @@ private static TimeZoneInfoResult TryGetTimeZoneFromLocalMachineCore(string id, value = null; e = null; + byte[]? rawData=null; +#if TARGET_WASI || TARGET_BROWSER + if (UseEmbeddedTzDatabase) + { + if(!TryLoadEmbeddedTzFile(id, ref rawData)) + { + e = new FileNotFoundException(id, "Embedded TZ data not found"); + return TimeZoneInfoResult.TimeZoneNotFoundException; + } + + value = GetTimeZoneFromTzData(rawData, id); + + if (value == null) + { + e = new InvalidTimeZoneException(SR.Format(SR.InvalidTimeZone_InvalidFileData, id, id)); + return TimeZoneInfoResult.InvalidTimeZoneException; + } + + return TimeZoneInfoResult.Success; + } +#endif + string timeZoneDirectory = GetTimeZoneDirectory(); string timeZoneFilePath = Path.Combine(timeZoneDirectory, id); - byte[] rawData; try { rawData = File.ReadAllBytes(timeZoneFilePath); @@ -74,52 +100,68 @@ private static TimeZoneInfoResult TryGetTimeZoneFromLocalMachineCore(string id, /// /// Lines that start with # are comments and are skipped. /// - private static List GetTimeZoneIds() + private static IEnumerable GetTimeZoneIds() + { +#if TARGET_WASI || TARGET_BROWSER + byte[]? rawData = null; + if (UseEmbeddedTzDatabase) + { + if(!TryLoadEmbeddedTzFile(TimeZoneFileName, ref rawData)) + { + return Array.Empty(); + } + using var reader = new StreamReader(new MemoryStream(rawData), Encoding.UTF8); + return ParseTimeZoneIds(reader); + } +#endif + try + { + using var reader = new StreamReader(Path.Combine(GetTimeZoneDirectory(), TimeZoneFileName), Encoding.UTF8); + return ParseTimeZoneIds(reader); + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + return Array.Empty(); + } + + private static List ParseTimeZoneIds(StreamReader reader) { List timeZoneIds = new List(); - try + string? zoneTabFileLine; + while ((zoneTabFileLine = reader.ReadLine()) != null) { - using (StreamReader sr = new StreamReader(Path.Combine(GetTimeZoneDirectory(), TimeZoneFileName), Encoding.UTF8)) + if (!string.IsNullOrEmpty(zoneTabFileLine) && zoneTabFileLine[0] != '#') { - string? zoneTabFileLine; - while ((zoneTabFileLine = sr.ReadLine()) != null) + // the format of the line is "country-code \t coordinates \t TimeZone Id \t comments" + + int firstTabIndex = zoneTabFileLine.IndexOf('\t'); + if (firstTabIndex >= 0) { - if (!string.IsNullOrEmpty(zoneTabFileLine) && zoneTabFileLine[0] != '#') + int secondTabIndex = zoneTabFileLine.IndexOf('\t', firstTabIndex + 1); + if (secondTabIndex >= 0) { - // the format of the line is "country-code \t coordinates \t TimeZone Id \t comments" - - int firstTabIndex = zoneTabFileLine.IndexOf('\t'); - if (firstTabIndex >= 0) + string timeZoneId; + int startIndex = secondTabIndex + 1; + int thirdTabIndex = zoneTabFileLine.IndexOf('\t', startIndex); + if (thirdTabIndex >= 0) { - int secondTabIndex = zoneTabFileLine.IndexOf('\t', firstTabIndex + 1); - if (secondTabIndex >= 0) - { - string timeZoneId; - int startIndex = secondTabIndex + 1; - int thirdTabIndex = zoneTabFileLine.IndexOf('\t', startIndex); - if (thirdTabIndex >= 0) - { - int length = thirdTabIndex - startIndex; - timeZoneId = zoneTabFileLine.Substring(startIndex, length); - } - else - { - timeZoneId = zoneTabFileLine.Substring(startIndex); - } + int length = thirdTabIndex - startIndex; + timeZoneId = zoneTabFileLine.Substring(startIndex, length); + } + else + { + timeZoneId = zoneTabFileLine.Substring(startIndex); + } - if (!string.IsNullOrEmpty(timeZoneId)) - { - timeZoneIds.Add(timeZoneId); - } - } + if (!string.IsNullOrEmpty(timeZoneId)) + { + timeZoneIds.Add(timeZoneId); } } } } } - catch (IOException) { } - catch (UnauthorizedAccessException) { } return timeZoneIds; } @@ -379,6 +421,22 @@ private static bool TryLoadTzFile(string tzFilePath, [NotNullWhen(true)] ref byt return false; } +#if TARGET_WASI || TARGET_BROWSER + private static bool TryLoadEmbeddedTzFile(string name, [NotNullWhen(true)] ref byte[]? rawData) + { + IntPtr bytes = Interop.Sys.GetTimeZoneData(name, out int length); + if(bytes == IntPtr.Zero) + { + rawData = null; + return false; + } + + rawData = new byte[length]; + Marshal.Copy(bytes, rawData, 0, length); + return true; + } +#endif + /// /// Gets the tzfile raw data for the current 'local' time zone using the following rules. /// @@ -387,6 +445,10 @@ private static bool TryLoadTzFile(string tzFilePath, [NotNullWhen(true)] ref byt /// 2. Get the default TZ from the device /// 3. Use UTC if all else fails. /// + /// On WASI / Browser + /// 0. if TZDIR is not set, use TZ variable as id to embedded database. + /// 1. fall back to unix behavior if TZDIR is set. + /// /// On all other platforms /// 1. Read the TZ environment variable. If it is set, use it. /// 2. Look for the data in /etc/localtime. @@ -406,6 +468,11 @@ private static bool TryGetLocalTzFile([NotNullWhen(true)] out byte[]? rawData, [ { #if TARGET_IOS || TARGET_TVOS tzVariable = Interop.Sys.GetDefaultTimeZone(); +#elif TARGET_WASI || TARGET_BROWSER + if (UseEmbeddedTzDatabase) + { + return false; // use UTC + } #else return TryLoadTzFile("/etc/localtime", ref rawData, ref id) || @@ -419,6 +486,17 @@ private static bool TryGetLocalTzFile([NotNullWhen(true)] out byte[]? rawData, [ { return false; } +#if TARGET_WASI || TARGET_BROWSER + if (UseEmbeddedTzDatabase) + { + if(!TryLoadEmbeddedTzFile(tzVariable, ref rawData)) + { + return false; + } + id = tzVariable; + return true; + } +#endif // Otherwise, use the path from the env var. If it's not absolute, make it relative // to the system timezone directory diff --git a/src/mono/mono/utils/mono-dl-wasm.c b/src/mono/mono/utils/mono-dl-wasm.c index 15189fe9db0cb..774d1ba6dd2b1 100644 --- a/src/mono/mono/utils/mono-dl-wasm.c +++ b/src/mono/mono/utils/mono-dl-wasm.c @@ -1,5 +1,6 @@ #include #include +#include #if defined (HOST_WASM) @@ -93,3 +94,45 @@ mono_dl_close_handle (MonoDl *module, MonoError *error) MONO_EMPTY_SOURCE_FILE (mono_dl_wasm); #endif + +#if defined (HOST_WASM) + +static GHashTable *name_to_blob = NULL; + +typedef struct { + const unsigned char *data; + unsigned int size; +} FileBlob; + +int +mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size) +{ + // printf("mono_wasm_add_bundled_file: %s %p %d\n", name, data, size); + if(name_to_blob == NULL) + { + name_to_blob = g_hash_table_new (g_str_hash, g_str_equal); + } + FileBlob *blob = g_new0 (FileBlob, 1); + blob->data = data; + blob->size = size; + g_hash_table_insert (name_to_blob, (gpointer) name, blob); + return 0; +} + +const unsigned char* +mono_wasm_get_bundled_file (const char *name, int* out_length) +{ + FileBlob *blob = (FileBlob *)g_hash_table_lookup (name_to_blob, name); + if (blob != NULL) + { + // printf("mono_wasm_get_bundled_file: %s %p %d \n", name, blob->data, blob->size); + *out_length = blob->size; + return blob->data; + } + + // printf("mono_wasm_get_bundled_file: %s not found \n", name); + *out_length = 0; + return NULL; +} + +#endif /* HOST_WASM */ diff --git a/src/mono/mono/utils/mono-dl.h b/src/mono/mono/utils/mono-dl.h index 40d5fff11d094..d3893ff8a7139 100644 --- a/src/mono/mono/utils/mono-dl.h +++ b/src/mono/mono/utils/mono-dl.h @@ -58,5 +58,10 @@ int mono_dl_convert_flags (int mono_flags, int native_flags); char* mono_dl_current_error_string (void); const char* mono_dl_get_system_dir (void); +#if defined (HOST_WASM) +int mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size); +const unsigned char* mono_wasm_get_bundled_file (const char *name, int* out_length); +#endif /* HOST_WASM */ + #endif /* __MONO_UTILS_DL_H__ */ diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 2b6fff044afd0..7c863a357f8e2 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -1,7 +1,6 @@ - @@ -169,6 +168,7 @@ <_WasmCommonCFlags Include="-DGEN_PINVOKE=1" /> + <_WasmCommonCFlags Include="-DGEN_ASSEMBLIES=1" /> @@ -341,40 +341,38 @@ + - - <_GetBundledFileSourcePath>$(_WasmIntermediateOutputPath)dotnet_wasi_getbundledfile.c - - - - + - - $(_WasmIntermediateOutputPath)%(WasmBundleFilesWithHashes.Filename)%(WasmBundleFilesWithHashes.Extension).$([System.String]::Copy(%(WasmBundleFilesWithHashes.FileHash)).Substring(0, 8)).o - + + $(_WasmIntermediateOutputPath)%(WasmBundleAssembliesWithHashes.Filename)%(WasmBundleAssembliesWithHashes.Extension).$([System.String]::Copy(%(WasmBundleAssembliesWithHashes.FileHash)).Substring(0, 8)).o + - - - - - - + + + + - <_WasiObjectFilesForBundle Include="$(_GetBundledFileSourcePath)" /> - <_WasiObjectFilesForBundle Include="%(WasmBundleFilesWithHashes.ObjectFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)wasi_bundled_assemblies.o" /> + <_WasiObjectFilesForBundle Include="%(WasmBundleAssembliesWithHashes.DestinationFile)" /> - + + diff --git a/src/mono/wasi/build/WasiApp.targets b/src/mono/wasi/build/WasiApp.targets index 0688e932d0e16..71f5ba1b27c34 100644 --- a/src/mono/wasi/build/WasiApp.targets +++ b/src/mono/wasi/build/WasiApp.targets @@ -329,7 +329,6 @@ - diff --git a/src/mono/wasi/build/WasiSdk.Defaults.props b/src/mono/wasi/build/WasiSdk.Defaults.props index 51ce6b50a2eff..67b77eaba3eb9 100644 --- a/src/mono/wasi/build/WasiSdk.Defaults.props +++ b/src/mono/wasi/build/WasiSdk.Defaults.props @@ -3,5 +3,5 @@ $(WASI_SDK_PATH) $(WasiSdkRoot)\bin\clang $(WasiClang).exe - + diff --git a/src/mono/wasi/runtime/CMakeLists.txt b/src/mono/wasi/runtime/CMakeLists.txt index 8e4114f0c31e4..d3da8472dd6ea 100644 --- a/src/mono/wasi/runtime/CMakeLists.txt +++ b/src/mono/wasi/runtime/CMakeLists.txt @@ -23,6 +23,7 @@ target_link_libraries(dotnet ${MONO_ARTIFACTS_DIR}/libmono-ee-interp.a ${MONO_ARTIFACTS_DIR}/libmonosgen-2.0.a ${MONO_ARTIFACTS_DIR}/libmono-icall-table.a + ${NATIVE_BIN_DIR}/wasm-bundled-timezones.a ${NATIVE_BIN_DIR}/libSystem.Native.a ${NATIVE_BIN_DIR}/libSystem.IO.Compression.Native.a ) diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 1689d36ab244a..923e914b31281 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -64,8 +64,11 @@ int32_t monoeg_g_hasenv(const char *variable); void mono_free (void*); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); -extern const char* dotnet_wasi_getbundledfile(const char* name, int* out_length); -extern void dotnet_wasi_registerbundledassemblies(); +extern void mono_wasm_register_bundle_timezones(); +#ifdef GEN_ASSEMBLIES +extern void mono_wasm_register_bundle_assemblies(); +#endif + extern const char* dotnet_wasi_getentrypointassemblyname(); int mono_wasm_enable_gc = 1; @@ -401,6 +404,10 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); + mono_wasm_register_bundle_timezones(); +#ifdef GEN_ASSEMBLIES + mono_wasm_register_bundle_assemblies(); +#endif mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); diff --git a/src/mono/wasi/runtime/main.c b/src/mono/wasi/runtime/main.c index 3b0e5a322864d..ff0a25e414406 100644 --- a/src/mono/wasi/runtime/main.c +++ b/src/mono/wasi/runtime/main.c @@ -5,18 +5,12 @@ // This symbol's implementation is generated during the build const char* dotnet_wasi_getentrypointassemblyname(); -// These are generated by EmitWasmBundleObjectFile -const char* dotnet_wasi_getbundledfile(const char* name, int* out_length); -void dotnet_wasi_registerbundledassemblies(); - #ifdef WASI_AFTER_RUNTIME_LOADED_DECLARATIONS // This is supplied from the MSBuild itemgroup @(WasiAfterRuntimeLoaded) WASI_AFTER_RUNTIME_LOADED_DECLARATIONS #endif int main(int argc, char * argv[]) { - // generated during the build - dotnet_wasi_registerbundledassemblies(); #ifdef WASI_AFTER_RUNTIME_LOADED_CALLS // This is supplied from the MSBuild itemgroup @(WasiAfterRuntimeLoaded) diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index 387af89db0951..1f739bae0654d 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -10,11 +10,16 @@ $([MSBuild]::NormalizeDirectory('$(PkgMicrosoft_NETCore_Runtime_ICU_Transport)', 'runtimes', 'wasi-wasm-threads', 'native', 'lib')) false false - $(ArtifactsObjDir)wasm + $(ArtifactsObjDir)wasi <_WasiDefaultsRspPath>$(NativeBinDir)src\wasi-default.rsp <_WasiCompileRspPath>$(NativeBinDir)src\wasi-compile.rsp <_WasiLinkRspPath>$(NativeBinDir)src\wasi-link.rsp false + $(WASI_SDK_PATH) + $(WasiSdkRoot)\bin\clang + $(WasiClang).exe + $(WasiSdkRoot)\bin\llvm-ar + $(WasiLLVMAr).exe @@ -28,23 +33,12 @@ - - - - $(NativeBinDir)dotnet.timezones.blat - - - - - $(WasmObjDir)\pinvoke-table.h - $(WasmObjDir)\wasm_m2n_invoke.g.h + $(WasiObjDir)\pinvoke-table.h + $(WasiObjDir)\wasm_m2n_invoke.g.h @@ -62,7 +56,7 @@ - + + + + + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) + <_WasmTimezonesBundleObjectFile>$(WasiObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundleArchive>$(WasiObjDir)\wasm-bundled-timezones.a + <_WasmTimezonesArchiveRsp>$(WasiObjDir)\wasm-bundled-timezones-archive.rsp + + + <_WasmTimezonesInternal Include="$(_WasmTimezonesPath)\**\*.*" WasmRole="Timezone"/> + + + + + + + $(WasiObjDir)\wasm-bundled-$([System.String]::Copy(%(WasmBundleTimezonesWithHashes.FileHash))).o + $([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) + + + + + + + + + + + + + + <_WasmArchivedTimezones Include="$(_WasmTimezonesBundleArchive)" /> + + + + + + + + + + @@ -145,7 +190,7 @@ + DependsOnTargets="GenerateWasiPropsAndRspFiles;GenerateManagedToNative;GenerateTimezonesArchive"> - $(ArtifactsObjDir)wasm/pinvoke-table.h - $(ArtifactsObjDir)wasm/wasm_m2n_invoke.g.h + $(ArtifactsObjDir)wasi/pinvoke-table.h + $(ArtifactsObjDir)wasi/wasm_m2n_invoke.g.h $([MSBuild]::EnsureTrailingSlash('$(WASI_SDK_PATH)').Replace('\', '/')) -g -Os -DDEBUG=1 -DENABLE_AOT_PROFILER=1 @@ -199,9 +244,9 @@ <_FilesToCopy Include="$(MSBuildThisFileDirectory)runtime/stubs.c" DestinationFolder="$(NativeBinDir)src" /> <_FilesToCopy Include="$(MSBuildThisFileDirectory)runtime/synthetic-pthread.c" DestinationFolder="$(NativeBinDir)src" /> - <_FilesToCopy Include="$(MonoProjectRoot)wasi\mono-include\driver.h" DestinationFolder="$(NativeBinDir)include\wasm" /> - <_FilesToCopy Include="$(MonoProjectRoot)wasi\mono-include\pinvoke.h" DestinationFolder="$(NativeBinDir)include\wasm" /> - <_FilesToCopy Include="$(MonoProjectRoot)wasm\runtime\gc-common.h" DestinationFolder="$(NativeBinDir)include\wasm" /> + <_FilesToCopy Include="$(MonoProjectRoot)wasi\mono-include\driver.h" DestinationFolder="$(NativeBinDir)include\wasm" /> + <_FilesToCopy Include="$(MonoProjectRoot)wasi\mono-include\pinvoke.h" DestinationFolder="$(NativeBinDir)include\wasm" /> + <_FilesToCopy Include="$(MonoProjectRoot)wasm\runtime\gc-common.h" DestinationFolder="$(NativeBinDir)include\wasm" /> @@ -230,12 +276,11 @@ - - diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 714ad4075bb4d..2692524c7715f 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -641,7 +641,6 @@ static void AssertRuntimePackPath(string buildOutput, string targetFramework) { "index.html", mainJS, - "dotnet.timezones.blat", "dotnet.wasm", "mono-config.json", "dotnet.js" diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 6b68828e6685b..74d7b35dc83cc 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -335,7 +335,6 @@ Exists('$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.js.symbols')" /> - diff --git a/src/mono/wasm/runtime/CMakeLists.txt b/src/mono/wasm/runtime/CMakeLists.txt index dab32f84e060c..afd0cdd90dd68 100644 --- a/src/mono/wasm/runtime/CMakeLists.txt +++ b/src/mono/wasm/runtime/CMakeLists.txt @@ -26,6 +26,7 @@ target_link_libraries(dotnet ${MONO_ARTIFACTS_DIR}/libmono-wasm-eh-js.a ${MONO_ARTIFACTS_DIR}/libmono-profiler-aot.a ${MONO_ARTIFACTS_DIR}/libmono-profiler-browser.a + ${NATIVE_BIN_DIR}/wasm-bundled-timezones.a ${NATIVE_BIN_DIR}/libSystem.Native.a ${NATIVE_BIN_DIR}/libSystem.IO.Compression.Native.a) diff --git a/src/mono/wasm/runtime/assets.ts b/src/mono/wasm/runtime/assets.ts index e9b98a8a242da..2b626c5177b20 100644 --- a/src/mono/wasm/runtime/assets.ts +++ b/src/mono/wasm/runtime/assets.ts @@ -481,11 +481,6 @@ export function mono_wasm_load_data_archive(data: Uint8Array, prefix: string): b data = data.slice(manifestSize + 8); // Create the folder structure - // /usr/share/zoneinfo - // /usr/share/zoneinfo/Africa - // /usr/share/zoneinfo/Asia - // .. - const folders = new Set(); manifest.filter(m => { const file = m[0]; diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index 9b145963b0d4c..b3ca85f754674 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -68,6 +68,7 @@ void mono_free (void*); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); char *mono_method_full_name (MonoMethod *method, int signature); +extern void mono_wasm_register_bundle_timezones(); static void mono_wasm_init_finalizer_thread (void); @@ -537,6 +538,8 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); + + mono_wasm_register_bundle_timezones(); mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); diff --git a/src/mono/wasm/runtime/startup.ts b/src/mono/wasm/runtime/startup.ts index 3fea12709290a..708968908be38 100644 --- a/src/mono/wasm/runtime/startup.ts +++ b/src/mono/wasm/runtime/startup.ts @@ -488,11 +488,15 @@ async function instantiate_wasm_module( async function _apply_configuration_from_args() { try { const tz = Intl.DateTimeFormat().resolvedOptions().timeZone; - mono_wasm_setenv("TZ", tz || "UTC"); + if (tz) mono_wasm_setenv("TZ", tz); } catch { - mono_wasm_setenv("TZ", "UTC"); + // no action } + // create /usr/share folder which is SpecialFolder.CommonApplicationData + Module["FS_createPath"]("/", "usr", true, true); + Module["FS_createPath"]("/", "usr/share", true, true); + for (const k in config.environmentVariables) { const v = config.environmentVariables![k]; if (typeof (v) === "string") diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index ff6737584da3c..e2e35c3565e73 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -34,6 +34,8 @@ <_EmccLinkRspPath>$(NativeBinDir)src\emcc-link.rsp <_EmccLinkUndefinedSymbolsFile>$(WasmObjDir)src\symbols.undefined false + $(EMSDK_PATH)\upstream\bin\llvm-ar + $(EmSdkLLVMAr).exe @@ -47,17 +49,6 @@ - - - - $(NativeBinDir)dotnet.timezones.blat - - - - @@ -89,6 +80,82 @@ + + + + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) + <_WasmTimezonesBundleSourceFile>$(WasmObjDir)\wasm-bundled-timezones.c + <_WasmTimezonesBundleObjectFile>$(WasmObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundleArchive>$(WasmObjDir)\wasm-bundled-timezones.a + <_WasmTimezonesSourcesRsp>$(WasmObjDir)\wasm-bundled-timezones-sources.rsp + <_WasmTimezonesArchiveRsp>$(WasmObjDir)\wasm-bundled-timezones-archive.rsp + + + <_WasmTimezonesInternal Include="$(_WasmTimezonesPath)\**\*.*" WasmRole="Timezone"/> + + + + + + + $(WasmObjDir)\wasm-bundled-$([System.String]::Copy(%(WasmBundleTimezonesWithHashes.FileHash))).c + $(WasmObjDir)\wasm-bundled-$([System.String]::Copy(%(WasmBundleTimezonesWithHashes.FileHash))).o + $([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) + + + + + + + + + + + + + + + + + + + + + + <_WasmArchivedTimezones Include="$(WasmObjDir)\wasm-bundled-timezones.a" /> + + + + + + + + + + + + + + @@ -249,7 +316,7 @@ + DependsOnTargets="GenerateEmccPropsAndRspFiles;GenerateManagedToNative;GenerateTimezonesArchive;InstallNpmPackages;BuildWithRollup"> @@ -347,8 +415,7 @@ $(NativeBinDir)dotnet.d.ts; $(NativeBinDir)dotnet-legacy.d.ts; $(NativeBinDir)package.json; - $(NativeBinDir)dotnet.wasm; - $(NativeBinDir)dotnet.timezones.blat" + $(NativeBinDir)dotnet.wasm" DestinationFolder="$(MicrosoftNetCoreAppRuntimePackNativeDir)" SkipUnchangedFiles="true" /> @@ -361,7 +428,7 @@ DestinationFolder="$(MicrosoftNetCoreAppRuntimePackNativeDir)" SkipUnchangedFiles="true" /> - diff --git a/src/native/libs/System.Native/entrypoints.c b/src/native/libs/System.Native/entrypoints.c index c4faf76ec7dac..6b66bd8eeb40f 100644 --- a/src/native/libs/System.Native/entrypoints.c +++ b/src/native/libs/System.Native/entrypoints.c @@ -48,6 +48,7 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_GetSignalForBreak) DllImportEntry(SystemNative_SetSignalForBreak) DllImportEntry(SystemNative_GetSystemTimeAsTicks) + DllImportEntry(SystemNative_GetTimeZoneData) DllImportEntry(SystemNative_ConvertErrorPlatformToPal) DllImportEntry(SystemNative_ConvertErrorPalToPlatform) DllImportEntry(SystemNative_StrErrorR) diff --git a/src/native/libs/System.Native/pal_datetime.c b/src/native/libs/System.Native/pal_datetime.c index 73c0a2583e450..c6f1373eb4e9f 100644 --- a/src/native/libs/System.Native/pal_datetime.c +++ b/src/native/libs/System.Native/pal_datetime.c @@ -3,6 +3,7 @@ #include "pal_config.h" #include "pal_datetime.h" +#include "pal_utilities.h" #include #include #include @@ -19,6 +20,10 @@ static const int64_t NANOSECONDS_PER_TICK = 100; static const int64_t TICKS_PER_MICROSECOND = 10; /* 1000 / 100 */ #endif +#if defined(TARGET_WASI) || defined(TARGET_BROWSER) +extern const unsigned char* mono_wasm_get_bundled_file(const char* name, int* out_length); +#endif + // // SystemNative_GetSystemTimeAsTicks return the system time as ticks (100 nanoseconds) // since 00:00 01 January 1970 UTC (Unix epoch) @@ -56,3 +61,18 @@ char* SystemNative_GetDefaultTimeZone(void) } } #endif + + +const char* SystemNative_GetTimeZoneData(const char* name, int* length) +{ + assert(name != NULL); + assert(length != NULL); +#if defined(TARGET_WASI) || defined(TARGET_BROWSER) + return (const char*) mono_wasm_get_bundled_file(name, length); +#else + assert_msg(false, "Not supported on this platform", 0); + (void)name; // unused + (void)length; // unused + return NULL; +#endif +} diff --git a/src/native/libs/System.Native/pal_datetime.h b/src/native/libs/System.Native/pal_datetime.h index bc59762a066b0..b6da4a89cc327 100644 --- a/src/native/libs/System.Native/pal_datetime.h +++ b/src/native/libs/System.Native/pal_datetime.h @@ -11,3 +11,5 @@ PALEXPORT int64_t SystemNative_GetSystemTimeAsTicks(void); #if defined(TARGET_ANDROID) || defined(__APPLE__) PALEXPORT char* SystemNative_GetDefaultTimeZone(void); #endif + +PALEXPORT const char* SystemNative_GetTimeZoneData(const char* name, int* length); diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 8ed6c2c13711e..91e5b77c1e198 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -113,7 +113,8 @@ static string CreateTemporaryBatchFile(string command) bool silent = true, bool logStdErrAsMessage = false, MessageImportance debugMessageImportance=MessageImportance.High, - string? label=null) + string? label=null, + Action? inputProvider = null) { string msgPrefix = label == null ? string.Empty : $"[{label}] "; logger.LogMessage(debugMessageImportance, $"{msgPrefix}Running: {path} {args}"); @@ -125,6 +126,7 @@ static string CreateTemporaryBatchFile(string command) CreateNoWindow = true, RedirectStandardError = true, RedirectStandardOutput = true, + RedirectStandardInput = inputProvider != null, Arguments = args, }; @@ -181,6 +183,7 @@ static string CreateTemporaryBatchFile(string command) }; process.BeginOutputReadLine(); process.BeginErrorReadLine(); + inputProvider?.Invoke(process.StandardInput.BaseStream); process.WaitForExit(); logger.LogMessage(debugMessageImportance, $"{msgPrefix}Exit code: {process.ExitCode}"); diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleFiles.cs b/src/tasks/WasmAppBuilder/EmitWasmBundleFiles.cs new file mode 100644 index 0000000000000..e7780f0ee2066 --- /dev/null +++ b/src/tasks/WasmAppBuilder/EmitWasmBundleFiles.cs @@ -0,0 +1,268 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; + +namespace Microsoft.WebAssembly.Build.Tasks; + +public class EmitWasmBundleFiles : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource BuildTaskCancelled { get; } = new(); + + [Required] + public ITaskItem[] FilesToBundle { get; set; } = default!; + + // if not provided the task will generate source files to disk + public string? ClangExecutable { get; set; } + + [Required] + public string BundleName { get; set; } = default!; + + [Required] + public string BundleFile { get; set; } = default!; + + [Required] + public string RegistrationCallbackFunctionName { get; set; } = default!; + + public override bool Execute() + { + if (!string.IsNullOrEmpty(ClangExecutable) && !File.Exists(ClangExecutable)) + { + Log.LogError($"Cannot find {nameof(ClangExecutable)}={ClangExecutable}"); + return false; + } + Action> emitter = string.IsNullOrEmpty(ClangExecutable) ? WriteSource : Compile; + + // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore + // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. + var filesToBundleByDestinationFileName = FilesToBundle.GroupBy(f => f.GetMetadata("DestinationFile")).ToList(); + + // We're handling the incrementalism within this task, because it needs to be based on file content hashes + // and not on timetamps. The output filenames contain a content hash, so if any such file already exists on + // disk with that name, we know it must be up-to-date. + var remainingDestinationFilesToBundle = filesToBundleByDestinationFileName.Where(g => !File.Exists(g.Key)).ToArray(); + + // If you're only touching the leaf project, we don't really need to tell you that. + // But if there's more work to do it's valuable to show progress. + var verbose = remainingDestinationFilesToBundle.Length > 1; + var verboseCount = 0; + + if (remainingDestinationFilesToBundle.Length > 0) + { + int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + + Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, i => + { + var group = remainingDestinationFilesToBundle[i]; + + // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, + // since we know each group's ITaskItems all contain the same binary data + var contentSourceFile = group.First(); + + var outputFile = group.Key; + var inputFile = contentSourceFile.ItemSpec; + if (verbose) + { + var registeredName = contentSourceFile.GetMetadata("RegisteredName"); + if(string.IsNullOrEmpty(registeredName)) + { + registeredName = Path.GetFileName(inputFile); + } + var count = Interlocked.Increment(ref verboseCount); + Log.LogMessage(MessageImportance.High, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); + } + + Log.LogMessage(MessageImportance.Low, "Bundling {0} as {1}", inputFile, outputFile); + var symbolName = ToSafeSymbolName(outputFile); + emitter(outputFile, (codeStream) => { + using var inputStream = File.OpenRead(inputFile); + BundleFileToCSource(symbolName, inputStream, codeStream); + }); + }); + } + + var filesToBundleByRegisteredName = FilesToBundle.GroupBy(file => { + var registeredName = file.GetMetadata("RegisteredName"); + if(string.IsNullOrEmpty(registeredName)) + { + registeredName = Path.GetFileName(file.ItemSpec); + } + return registeredName; + }).ToList(); + + var files = filesToBundleByRegisteredName.Select(group => { + var registeredFile = group.First(); + var outputFile = registeredFile.GetMetadata("DestinationFile"); + var registeredName = group.Key; + var symbolName = ToSafeSymbolName(outputFile); + return (registeredName, symbolName); + }).ToList(); + + Log.LogMessage(MessageImportance.High, "Bundling {2} objects into mono_wasm_register_bundle_{0} as {1}", BundleName, BundleFile, files.Count); + emitter(BundleFile, (inputStream) => + { + using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); + GenerateRegisterBundledObjects("mono_wasm_register_bundle_" + BundleName, RegistrationCallbackFunctionName, files, outputUtf8Writer); + }); + + return !Log.HasLoggedErrors; + } + + + public void Cancel() + { + BuildTaskCancelled.Cancel(); + } + + #region Helpers + + private static readonly Encoding Utf8NoBom = new UTF8Encoding(false); + private static readonly byte[] HexToUtf8Lookup = InitLookupTable(); + private static readonly byte[] NewLineAndIndentation = new[] { (byte)0x0a, (byte)0x20, (byte)0x20 }; + + private static byte[] InitLookupTable() + { + // Every 6 bytes in this array represents the output for a different input byte value. + // For example, the input byte 0x1a (26 decimal) corresponds to bytes 156-161 (26*6=156), + // whose values will be ['0', 'x', '1', 'a', ',', ' '], which is the UTF-8 representation + // for "0x1a, ". This is just a faster alternative to calling .ToString("x2") on every + // byte of the input file and then pushing that string through UTF8Encoding. + var lookup = new byte[256 * 6]; + for (int i = 0; i < 256; i++) + { + string byteAsHex = i.ToString("x2"); + char highOrderChar = BitConverter.IsLittleEndian ? byteAsHex[0] : byteAsHex[1]; + char lowOrderChar = BitConverter.IsLittleEndian ? byteAsHex[1] : byteAsHex[0]; + lookup[i * 6 + 0] = (byte)'0'; + lookup[i * 6 + 1] = (byte)'x'; + lookup[i * 6 + 2] = (byte)highOrderChar; + lookup[i * 6 + 3] = (byte)lowOrderChar; + lookup[i * 6 + 4] = (byte)','; + lookup[i * 6 + 5] = (byte)' '; + } + + return lookup; + } + + public void WriteSource(string destinationFile, Action inputProvider) + { + using (var fileStream = File.Create(destinationFile)) + { + inputProvider(fileStream); + } + } + + public void Compile(string destinationFile, Action inputProvider) + { + if (Path.GetDirectoryName(destinationFile) is string destDir && !string.IsNullOrEmpty(destDir)) + Directory.CreateDirectory(destDir); + + (int exitCode, string output) = Utils.TryRunProcess(Log, + ClangExecutable!, + $"-xc -o \"{destinationFile}\" -c -", + null, null, true, false, MessageImportance.Low, null, + inputProvider); + if (exitCode != 0) + { + Log.LogError($"workload install failed with exit code {exitCode}: {output}"); + } + } + + public static void GenerateRegisterBundledObjects(string newFunctionName, string callbackFunctionName, ICollection<(string registeredName, string symbol)> files, StreamWriter outputUtf8Writer) + { + outputUtf8Writer.WriteLine($"int {callbackFunctionName}(const char* name, const unsigned char* data, unsigned int size);"); + outputUtf8Writer.WriteLine(); + + foreach (var tuple in files) + { + outputUtf8Writer.WriteLine($"extern const unsigned char {tuple.symbol}[];"); + outputUtf8Writer.WriteLine($"extern const int {tuple.symbol}_len;"); + } + + outputUtf8Writer.WriteLine(); + outputUtf8Writer.WriteLine($"void {newFunctionName}() {{"); + + foreach (var tuple in files) + { + outputUtf8Writer.WriteLine($" {callbackFunctionName} (\"{tuple.registeredName}\", {tuple.symbol}, {tuple.symbol}_len);"); + } + + outputUtf8Writer.WriteLine("}"); + } + + private static void BundleFileToCSource(string symbolName, FileStream inputStream, Stream outputStream) + { + // Emits a C source file in the same format as "xxd --include". Example: + // + // unsigned char Some_File_dll[] = { + // 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x0a + // }; + // unsigned int Some_File_dll_len = 6; + + var buf = new byte[4096]; + int bytesRead; + var generatedArrayLength = 0; + var bytesEmitted = 0; + + using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); + + outputUtf8Writer.Write($"unsigned char {symbolName}[] = {{"); + outputUtf8Writer.Flush(); + while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) + { + for (var i = 0; i < bytesRead; i++) + { + if (bytesEmitted++ % 12 == 0) + { + outputStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); + } + + var byteValue = buf[i]; + outputStream.Write(HexToUtf8Lookup, byteValue * 6, 6); + } + + generatedArrayLength += bytesRead; + } + + outputUtf8Writer.WriteLine("0\n};"); + outputUtf8Writer.WriteLine($"unsigned int {symbolName}_len = {generatedArrayLength};"); + outputUtf8Writer.Flush(); + outputStream.Flush(); + } + + private static string ToSafeSymbolName(string destinationFileName) + { + // Since destinationFileName includes a content hash, we can safely strip off the directory name + // as the filename is always unique enough. This avoid disclosing information about the build + // file structure in the resulting symbols. + var filename = Path.GetFileName(destinationFileName); + + // Equivalent to the logic from "xxd --include" + var sb = new StringBuilder(); + foreach (var c in filename) + { + sb.Append(IsAlphanumeric(c) ? c : '_'); + } + + return sb.ToString(); + } + + // Equivalent to "isalnum" + private static bool IsAlphanumeric(char c) => c + is (>= 'a' and <= 'z') + or (>= 'A' and <= 'Z') + or (>= '0' and <= '9'); + + #endregion + +} diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index c99a05445407c..dc0bd59ef26fd 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -310,7 +310,6 @@ protected override bool ExecuteInternal() if (!InvariantGlobalization) config.Assets.Add(new IcuData(IcuDataFileName!) { LoadRemote = RemoteSources?.Length > 0 }); - config.Assets.Add(new VfsEntry ("dotnet.timezones.blat") { VirtualPath = "/usr/share/zoneinfo/"}); config.Assets.Add(new WasmEntry ("dotnet.wasm") ); if (IncludeThreadsWorker) config.Assets.Add(new ThreadsWorkerEntry ("dotnet.worker.js") ); diff --git a/src/tasks/WasmAppBuilder/wasi/EmitWasmBundleObjectFile.cs b/src/tasks/WasmAppBuilder/wasi/EmitWasmBundleObjectFile.cs deleted file mode 100644 index 4a7e1009dd72d..0000000000000 --- a/src/tasks/WasmAppBuilder/wasi/EmitWasmBundleObjectFile.cs +++ /dev/null @@ -1,256 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Build.Framework; - -namespace Microsoft.WebAssembly.Build.Tasks; - -public class EmitWasmBundleObjectFile : Microsoft.Build.Utilities.Task, ICancelableTask -{ - private static readonly Encoding Utf8NoBom = new UTF8Encoding(false); - private static readonly byte[] HexToUtf8Lookup = InitLookupTable(); - private static readonly byte[] NewLineAndIndentation = new[] { (byte)0x0a, (byte)0x20, (byte)0x20 }; - private CancellationTokenSource BuildTaskCancelled { get; } = new(); - - // We only want to emit a single copy of the data for a given content hash, but we have to track all the - // different filenames that may be referencing that content - private ICollection> _filesToBundleByObjectFileName = default!; - - [Required] - public ITaskItem[] FilesToBundle { get; set; } = default!; - - [Required] - public string ClangExecutable { get; set; } = default!; - - [Output] - public string? BundleApiSourceCode { get; set; } - - private static byte[] InitLookupTable() - { - // Every 6 bytes in this array represents the output for a different input byte value. - // For example, the input byte 0x1a (26 decimal) corresponds to bytes 156-161 (26*6=156), - // whose values will be ['0', 'x', '1', 'a', ',', ' '], which is the UTF-8 representation - // for "0x1a, ". This is just a faster alternative to calling .ToString("x2") on every - // byte of the input file and then pushing that string through UTF8Encoding. - var lookup = new byte[256 * 6]; - for (int i = 0; i < 256; i++) - { - string byteAsHex = i.ToString("x2"); - char highOrderChar = BitConverter.IsLittleEndian ? byteAsHex[0] : byteAsHex[1]; - char lowOrderChar = BitConverter.IsLittleEndian ? byteAsHex[1] : byteAsHex[0]; - lookup[i * 6 + 0] = (byte)'0'; - lookup[i * 6 + 1] = (byte)'x'; - lookup[i * 6 + 2] = (byte)highOrderChar; - lookup[i * 6 + 3] = (byte)lowOrderChar; - lookup[i * 6 + 4] = (byte)','; - lookup[i * 6 + 5] = (byte)' '; - } - - return lookup; - } - - public override bool Execute() - { - if (!File.Exists(ClangExecutable)) - { - Log.LogError($"Cannot find {nameof(ClangExecutable)}={ClangExecutable}"); - return false; - } - - // The ObjectFile (output filename) already includes a content hash. Grouping by this filename therefore - // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. - _filesToBundleByObjectFileName = FilesToBundle.GroupBy(f => f.GetMetadata("ObjectFile")).ToList(); - - // We're handling the incrementalism within this task, because it needs to be based on file content hashes - // and not on timetamps. The output filenames contain a content hash, so if any such file already exists on - // disk with that name, we know it must be up-to-date. - var remainingObjectFilesToBundle = _filesToBundleByObjectFileName.Where(g => !File.Exists(g.Key)).ToArray(); - - // If you're only touching the leaf project, we don't really need to tell you that. - // But if there's more work to do it's valuable to show progress. - var verbose = remainingObjectFilesToBundle.Length > 1; - var verboseCount = 0; - - if (remainingObjectFilesToBundle.Length > 0) - { - int allowedParallelism = Math.Max(Math.Min(remainingObjectFilesToBundle.Length, Environment.ProcessorCount), 1); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); - - Parallel.For(0, remainingObjectFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, i => - { - var objectFile = remainingObjectFilesToBundle[i]; - - // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, - // since we know each group's ITaskItems all contain the same binary data - var contentSourceFile = objectFile.First(); - - var outputFile = objectFile.Key; - if (verbose) - { - var count = Interlocked.Increment(ref verboseCount); - Log.LogMessage(MessageImportance.High, "{0}/{1} Bundling {2}...", count, remainingObjectFilesToBundle.Length, Path.GetFileName(contentSourceFile.ItemSpec)); - } - - EmitObjectFile(contentSourceFile, outputFile); - }); - } - - BundleApiSourceCode = GetBundleFileApiSource(_filesToBundleByObjectFileName); - - return !Log.HasLoggedErrors; - } - - private void EmitObjectFile(ITaskItem fileToBundle, string destinationObjectFile) - { - Log.LogMessage(MessageImportance.Low, "Bundling {0} as {1}", fileToBundle.ItemSpec, destinationObjectFile); - - if (Path.GetDirectoryName(destinationObjectFile) is string destDir && !string.IsNullOrEmpty(destDir)) - Directory.CreateDirectory(destDir); - - var clangProcess = Process.Start(new ProcessStartInfo - { - FileName = ClangExecutable, - Arguments = $"-xc -o \"{destinationObjectFile}\" -c -", - RedirectStandardInput = true, - UseShellExecute = false, - })!; - - BundleFileToCSource(destinationObjectFile, fileToBundle, clangProcess.StandardInput.BaseStream); - clangProcess.WaitForExit(); - } - - private static string GetBundleFileApiSource(ICollection> bundledFilesByObjectFileName) - { - // Emit an object file that uses all the bundle file symbols and supplies an API - // for getting the bundled file data at runtime - var result = new StringBuilder(); - - result.AppendLine("#include "); - result.AppendLine(); - result.AppendLine("int mono_wasm_add_assembly(const char* name, const unsigned char* data, unsigned int size);"); - result.AppendLine(); - - foreach (var objectFileGroup in bundledFilesByObjectFileName) - { - var symbol = ToSafeSymbolName(objectFileGroup.Key); - result.AppendLine($"extern const unsigned char {symbol}[];"); - result.AppendLine($"extern const int {symbol}_len;"); - } - - result.AppendLine(); - result.AppendLine("const unsigned char* dotnet_wasi_getbundledfile(const char* name, int* out_length) {"); - - // TODO: Instead of a naive O(N) search through all bundled files, consider putting them in a - // hashtable or at least generating a series of comparisons equivalent to a binary search - - foreach (var objectFileGroup in bundledFilesByObjectFileName) - { - foreach (var file in objectFileGroup.Where(f => !string.Equals(f.GetMetadata("WasmRole"), "assembly", StringComparison.OrdinalIgnoreCase))) - { - var symbol = ToSafeSymbolName(objectFileGroup.Key); - result.AppendLine($" if (!strcmp (name, \"{file.ItemSpec.Replace("\\", "/")}\")) {{"); - result.AppendLine($" *out_length = {symbol}_len;"); - result.AppendLine($" return {symbol};"); - result.AppendLine(" }"); - result.AppendLine(); - } - } - - result.AppendLine(" return NULL;"); - result.AppendLine("}"); - - result.AppendLine(); - result.AppendLine("void dotnet_wasi_registerbundledassemblies() {"); - - foreach (var objectFileGroup in bundledFilesByObjectFileName) - { - foreach (var file in objectFileGroup.Where(f => string.Equals(f.GetMetadata("WasmRole"), "assembly", StringComparison.OrdinalIgnoreCase))) - { - var symbol = ToSafeSymbolName(objectFileGroup.Key); - result.AppendLine($" mono_wasm_add_assembly (\"{Path.GetFileName(file.ItemSpec)}\", {symbol}, {symbol}_len);"); - } - } - - result.AppendLine("}"); - - return result.ToString(); - } - - private static void BundleFileToCSource(string objectFileName, ITaskItem fileToBundle, Stream outputStream) - { - // Emits a C source file in the same format as "xxd --include". Example: - // - // unsigned char Some_File_dll[] = { - // 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x0a - // }; - // unsigned int Some_File_dll_len = 6; - - using var inputStream = File.OpenRead(fileToBundle.ItemSpec); - using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); - - var symbolName = ToSafeSymbolName(objectFileName); - outputUtf8Writer.Write($"unsigned char {symbolName}[] = {{"); - outputUtf8Writer.Flush(); - - var buf = new byte[4096]; - var bytesRead = 0; - var generatedArrayLength = 0; - var bytesEmitted = 0; - while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) - { - for (var i = 0; i < bytesRead; i++) - { - if (bytesEmitted++ % 12 == 0) - { - outputStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); - } - - var byteValue = buf[i]; - outputStream.Write(HexToUtf8Lookup, byteValue * 6, 6); - } - - generatedArrayLength += bytesRead; - } - - outputStream.Flush(); - outputUtf8Writer.WriteLine("0\n};"); - outputUtf8Writer.WriteLine($"unsigned int {symbolName}_len = {generatedArrayLength};"); - } - - private static string ToSafeSymbolName(string objectFileName) - { - // Since objectFileName includes a content hash, we can safely strip off the directory name - // as the filename is always unique enough. This avoid disclosing information about the build - // file structure in the resulting symbols. - var filename = Path.GetFileName(objectFileName); - - // Equivalent to the logic from "xxd --include" - var sb = new StringBuilder(); - foreach (var c in filename) - { - sb.Append(IsAlphanumeric(c) ? c : '_'); - } - - return sb.ToString(); - } - - // Equivalent to "isalnum" - private static bool IsAlphanumeric(char c) => c - is (>= 'a' and <= 'z') - or (>= 'A' and <= 'Z') - or (>= '0' and <= '9'); - - public void Cancel() - { - BuildTaskCancelled.Cancel(); - } -} diff --git a/src/tasks/WasmAppBuilder/wasi/WasmResolveAssemblyDependencies.cs b/src/tasks/WasmAppBuilder/wasi/WasmResolveAssemblyDependencies.cs deleted file mode 100644 index f0571c8eb3abf..0000000000000 --- a/src/tasks/WasmAppBuilder/wasi/WasmResolveAssemblyDependencies.cs +++ /dev/null @@ -1,199 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection.Metadata; -using System.Reflection.PortableExecutable; -using Microsoft.Build.Framework; -using TaskItem = Microsoft.Build.Utilities.TaskItem; - -namespace Microsoft.WebAssembly.Build.Tasks; - -/// -/// Starting from the entrypoint assembly, walks the graph of referenced assemblies using candidates from the -/// runtime pack (first priority) or application assembly list (second priority). This is a way of reducing the -/// number of bundled assemblies to the minimal set, instead of including every possible assembly from the runtime -/// pack and all framework references. -/// -public class WasmResolveAssemblyDependencies : Microsoft.Build.Utilities.Task -{ - [Required] - public string EntryPoint { get; set; } = default!; - - [Required] - public ITaskItem[] ApplicationAssemblies { get; set; } = default!; - - [Required] - public ITaskItem[] WasiRuntimePackAssemblies { get; set; } = default!; - - [Output] - public ITaskItem[]? Dependencies { get; set; } - - public override bool Execute() - { - var paths = ResolveRuntimeDependenciesCore(EntryPoint, ApplicationAssemblies, WasiRuntimePackAssemblies); - Dependencies = paths.Select(p => new TaskItem(p.Path)).ToArray(); - - return true; - } - - private static List ResolveRuntimeDependenciesCore( - string entryPointPath, - IEnumerable applicationAssemblies, - IEnumerable runtimePackAssemblies) - { - var entryAssembly = new AssemblyEntry(entryPointPath, GetAssemblyName(entryPointPath), originalTaskItem: null); - var applicationAssemblyEntries = CreateAssemblyLookup(applicationAssemblies); - var runtimePackAssemblyEntries = CreateAssemblyLookup(runtimePackAssemblies); - - var assemblyResolutionContext = new AssemblyResolutionContext( - entryAssembly, - applicationAssemblyEntries, - runtimePackAssemblyEntries); - assemblyResolutionContext.ResolveAssemblies(); - - return assemblyResolutionContext.Results; - } - - private static Dictionary CreateAssemblyLookup(IEnumerable assemblies) - { - var dictionary = new Dictionary(StringComparer.Ordinal); - foreach (var assembly in assemblies) - { - var assemblyName = GetAssemblyName(assembly.ItemSpec); - if (dictionary.TryGetValue(assemblyName, out var previous)) - { - throw new InvalidOperationException($"Multiple assemblies found with the same assembly name '{assemblyName}':" + - Environment.NewLine + string.Join(Environment.NewLine, previous, assembly.ItemSpec)); - } - dictionary[assemblyName] = new AssemblyEntry(assembly.ItemSpec, assemblyName, assembly); - } - - return dictionary; - } - - private static string GetAssemblyName(string assemblyPath) - { - // It would be more correct to return AssemblyName.GetAssemblyName(assemblyPath).Name, but that involves - // actually loading the assembly file and maybe hitting a BadImageFormatException if it's not actually - // something that can be loaded by the active .NET version (e.g., .NET Framework if this task is running - // inside VS). - // Instead we'll rely on the filename matching the assembly name. - return Path.GetFileNameWithoutExtension(assemblyPath); - } - - private sealed class AssemblyResolutionContext - { - public AssemblyResolutionContext( - AssemblyEntry entryAssembly, - Dictionary applicationAssemblies, - Dictionary runtimePackAssemblies) - { - EntryAssembly = entryAssembly; - ApplicationAssemblies = applicationAssemblies; - RuntimePackAssemblies = runtimePackAssemblies; - } - - public AssemblyEntry EntryAssembly { get; } - public Dictionary ApplicationAssemblies { get; } - public Dictionary RuntimePackAssemblies { get; } - - public List Results { get; } = new(); - - public void ResolveAssemblies() - { - var visitedAssemblies = new HashSet(); - var pendingAssemblies = new Stack(); - pendingAssemblies.Push(EntryAssembly.Name); - ResolveAssembliesCore(); - - void ResolveAssembliesCore() - { - while (pendingAssemblies.Count > 0) - { - var current = pendingAssemblies.Pop(); - if (visitedAssemblies.Add(current)) - { - // Not all references will be resolvable within the runtime pack. - // Skipping unresolved assemblies here is equivalent to passing "--skip-unresolved true" to the .NET linker. - if (Resolve(current) is AssemblyEntry resolved) - { - Results.Add(resolved); - var references = GetAssemblyReferences(resolved.Path); - foreach (var reference in references) - { - pendingAssemblies.Push(reference); - } - } - } - } - } - - AssemblyEntry? Resolve(string assemblyName) - { - if (string.Equals(assemblyName, EntryAssembly.Name, StringComparison.Ordinal)) - { - return EntryAssembly; - } - - // Resolution logic. For right now, we will prefer the runtime pack version of a given - // assembly if there is a candidate assembly and an equivalent runtime pack assembly. - if (RuntimePackAssemblies.TryGetValue(assemblyName, out var assembly) - || ApplicationAssemblies.TryGetValue(assemblyName, out assembly)) - { - return assembly; - } - - return null; - } - - static IReadOnlyList GetAssemblyReferences(string assemblyPath) - { - try - { - using var peReader = new PEReader(File.OpenRead(assemblyPath)); - if (!peReader.HasMetadata) - { - return Array.Empty(); // not a managed assembly - } - - var metadataReader = peReader.GetMetadataReader(); - - var references = new List(); - foreach (var handle in metadataReader.AssemblyReferences) - { - var reference = metadataReader.GetAssemblyReference(handle); - var referenceName = metadataReader.GetString(reference.Name); - - references.Add(referenceName); - } - - return references; - } - catch (BadImageFormatException) - { - // not a PE file, or invalid metadata - } - - return Array.Empty(); // not a managed assembly - } - } - } - - internal readonly struct AssemblyEntry - { - public AssemblyEntry(string path, string name, ITaskItem? originalTaskItem) - { - Path = path; - Name = name; - _originalTaskItem = originalTaskItem; - } - - private readonly ITaskItem? _originalTaskItem; - public string Path { get; } - public string Name { get; } - } -} diff --git a/src/tasks/WasmBuildTasks/GenerateWasmBundle.cs b/src/tasks/WasmBuildTasks/GenerateWasmBundle.cs deleted file mode 100644 index c4fc6a7849fc7..0000000000000 --- a/src/tasks/WasmBuildTasks/GenerateWasmBundle.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Buffers.Binary; -using System.Collections.Generic; -using System.IO; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Linq; -using System.Text.RegularExpressions; -using System.Net; -using System.Reflection; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -public class GenerateWasmBundle : Task -{ - [Required] - public string? InputDirectory { get; set; } - - [Required] - public string? OutputFileName { get; set; } - - private (byte[] json_bytes, MemoryStream stream) EnumerateData() - { - var indices = new List(); - var stream = new MemoryStream(); - - var directoryInfo = new DirectoryInfo(InputDirectory!); - - foreach (var entry in directoryInfo.EnumerateFiles("*", SearchOption.AllDirectories)) - { - var relativePath = Path.GetRelativePath(InputDirectory!, entry.FullName); - if (Path.DirectorySeparatorChar != '/') - relativePath = relativePath.Replace(Path.DirectorySeparatorChar, '/'); - - indices.Add(new object[] { relativePath, entry.Length }); - - using (var readStream = entry.OpenRead()) - readStream.CopyTo(stream); - } - - stream.Position = 0; - var jsonBytes = JsonSerializer.SerializeToUtf8Bytes(indices); - - return (jsonBytes, stream); - } - - public override bool Execute() - { - if (!Directory.Exists(InputDirectory)) - { - Log.LogError($"Input directory '{InputDirectory}' does not exist"); - return false; - } - - (byte[] json_bytes, MemoryStream stream) data = EnumerateData(); - - using (var file = File.Open(OutputFileName!, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite)) - { - var lengthBytes = new byte[4]; - var magicBytes = Encoding.ASCII.GetBytes("talb"); - BinaryPrimitives.WriteInt32LittleEndian(lengthBytes, data.json_bytes.Length); - file.Write(magicBytes); - file.Write(lengthBytes); - file.Write(data.json_bytes); - - data.stream.CopyTo(file); - } - - return true; - } -}