diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 4ff614b0c5061..8b2e6e69893de 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -192,7 +192,6 @@ $(LibrariesNativeArtifactsPath)package.json; $(LibrariesNativeArtifactsPath)dotnet.wasm; $(LibrariesNativeArtifactsPath)dotnet.js.symbols; - $(LibrariesNativeArtifactsPath)dotnet.timezones.blat; $(LibrariesNativeArtifactsPath)*.dat;" IsNative="true" /> @@ -225,7 +224,6 @@ diff --git a/eng/native/configurecompiler.cmake b/eng/native/configurecompiler.cmake index f5500b885df2e..c9a54547c0a20 100644 --- a/eng/native/configurecompiler.cmake +++ b/eng/native/configurecompiler.cmake @@ -586,6 +586,9 @@ if(CLR_CMAKE_TARGET_UNIX) add_compile_definitions($<$>>:TARGET_ANDROID>) elseif(CLR_CMAKE_TARGET_LINUX) add_compile_definitions($<$>>:TARGET_LINUX>) + if(CLR_CMAKE_TARGET_BROWSER) + add_compile_definitions($<$>>:TARGET_BROWSER>) + endif() if(CLR_CMAKE_TARGET_LINUX_MUSL) add_compile_definitions($<$>>:TARGET_LINUX_MUSL>) endif() @@ -599,8 +602,6 @@ if(CLR_CMAKE_TARGET_UNIX) endif() elseif(CLR_CMAKE_TARGET_WASI) add_compile_definitions($<$>>:TARGET_WASI>) -elseif(CLR_CMAKE_TARGET_BROWSER) - add_compile_definitions($<$>>:TARGET_BROWSER>) else(CLR_CMAKE_TARGET_UNIX) add_compile_definitions($<$>>:TARGET_WINDOWS>) endif(CLR_CMAKE_TARGET_UNIX) diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props index 5f56e0eb0bbea..848f426f9667c 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props @@ -221,13 +221,13 @@ + - diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs new file mode 100644 index 0000000000000..1444fb0d551b8 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class Sys + { + [LibraryImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_GetTimeZoneData", StringMarshalling = StringMarshalling.Utf8, SetLastError = true)] + internal static partial IntPtr GetTimeZoneData(string fileName, out int length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index ff1621fde13dd..bbef0e98c7da7 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -2143,6 +2143,9 @@ Common\Interop\Unix\System.Native\Interop.GetDefaultTimeZone.AnyMobile.cs + + Common\src\Interop\Unix\System.Native\Interop.GetTimeZoneData.Wasm.cs + Common\Interop\Unix\System.Native\Interop.GetEnv.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs b/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs index e9b915ed0fca1..d9651cfcd757d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs +++ b/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs @@ -18,6 +18,11 @@ public sealed partial class TimeZoneInfo private const string TimeZoneDirectoryEnvironmentVariable = "TZDIR"; private const string TimeZoneEnvironmentVariable = "TZ"; +#if TARGET_WASI || TARGET_BROWSER + // if TZDIR is set, then the embedded TZ data will be ignored and normal unix behavior will be used + private static readonly bool UseEmbeddedTzDatabase = string.IsNullOrEmpty(Environment.GetEnvironmentVariable(TimeZoneDirectoryEnvironmentVariable)); +#endif + private static TimeZoneInfo GetLocalTimeZoneCore() { // Without Registry support, create the TimeZoneInfo from a TZ file @@ -29,9 +34,31 @@ private static TimeZoneInfoResult TryGetTimeZoneFromLocalMachineCore(string id, value = null; e = null; + byte[]? rawData=null; string timeZoneDirectory = GetTimeZoneDirectory(); string timeZoneFilePath = Path.Combine(timeZoneDirectory, id); - byte[] rawData; + +#if TARGET_WASI || TARGET_BROWSER + if (UseEmbeddedTzDatabase) + { + if(!TryLoadEmbeddedTzFile(timeZoneFilePath, out rawData)) + { + e = new FileNotFoundException(id, "Embedded TZ data not found"); + return TimeZoneInfoResult.TimeZoneNotFoundException; + } + + value = GetTimeZoneFromTzData(rawData, id); + + if (value == null) + { + e = new InvalidTimeZoneException(SR.Format(SR.InvalidTimeZone_InvalidFileData, id, id)); + return TimeZoneInfoResult.InvalidTimeZoneException; + } + + return TimeZoneInfoResult.Success; + } +#endif + try { rawData = File.ReadAllBytes(timeZoneFilePath); @@ -74,52 +101,68 @@ private static TimeZoneInfoResult TryGetTimeZoneFromLocalMachineCore(string id, /// /// Lines that start with # are comments and are skipped. /// - private static List GetTimeZoneIds() + private static IEnumerable GetTimeZoneIds() + { + try + { + var fileName = Path.Combine(GetTimeZoneDirectory(), TimeZoneFileName); +#if TARGET_WASI || TARGET_BROWSER + if (UseEmbeddedTzDatabase) + { + if(!TryLoadEmbeddedTzFile(fileName, out var rawData)) + { + return Array.Empty(); + } + using var blobReader = new StreamReader(new MemoryStream(rawData), Encoding.UTF8); + return ParseTimeZoneIds(blobReader); + } +#endif + using var reader = new StreamReader(fileName, Encoding.UTF8); + return ParseTimeZoneIds(reader); + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + return Array.Empty(); + } + + private static List ParseTimeZoneIds(StreamReader reader) { List timeZoneIds = new List(); - try + string? zoneTabFileLine; + while ((zoneTabFileLine = reader.ReadLine()) != null) { - using (StreamReader sr = new StreamReader(Path.Combine(GetTimeZoneDirectory(), TimeZoneFileName), Encoding.UTF8)) + if (!string.IsNullOrEmpty(zoneTabFileLine) && zoneTabFileLine[0] != '#') { - string? zoneTabFileLine; - while ((zoneTabFileLine = sr.ReadLine()) != null) + // the format of the line is "country-code \t coordinates \t TimeZone Id \t comments" + + int firstTabIndex = zoneTabFileLine.IndexOf('\t'); + if (firstTabIndex >= 0) { - if (!string.IsNullOrEmpty(zoneTabFileLine) && zoneTabFileLine[0] != '#') + int secondTabIndex = zoneTabFileLine.IndexOf('\t', firstTabIndex + 1); + if (secondTabIndex >= 0) { - // the format of the line is "country-code \t coordinates \t TimeZone Id \t comments" - - int firstTabIndex = zoneTabFileLine.IndexOf('\t'); - if (firstTabIndex >= 0) + string timeZoneId; + int startIndex = secondTabIndex + 1; + int thirdTabIndex = zoneTabFileLine.IndexOf('\t', startIndex); + if (thirdTabIndex >= 0) { - int secondTabIndex = zoneTabFileLine.IndexOf('\t', firstTabIndex + 1); - if (secondTabIndex >= 0) - { - string timeZoneId; - int startIndex = secondTabIndex + 1; - int thirdTabIndex = zoneTabFileLine.IndexOf('\t', startIndex); - if (thirdTabIndex >= 0) - { - int length = thirdTabIndex - startIndex; - timeZoneId = zoneTabFileLine.Substring(startIndex, length); - } - else - { - timeZoneId = zoneTabFileLine.Substring(startIndex); - } + int length = thirdTabIndex - startIndex; + timeZoneId = zoneTabFileLine.Substring(startIndex, length); + } + else + { + timeZoneId = zoneTabFileLine.Substring(startIndex); + } - if (!string.IsNullOrEmpty(timeZoneId)) - { - timeZoneIds.Add(timeZoneId); - } - } + if (!string.IsNullOrEmpty(timeZoneId)) + { + timeZoneIds.Add(timeZoneId); } } } } } - catch (IOException) { } - catch (UnauthorizedAccessException) { } return timeZoneIds; } @@ -379,6 +422,22 @@ private static bool TryLoadTzFile(string tzFilePath, [NotNullWhen(true)] ref byt return false; } +#if TARGET_WASI || TARGET_BROWSER + private static bool TryLoadEmbeddedTzFile(string name, [NotNullWhen(true)] out byte[]? rawData) + { + IntPtr bytes = Interop.Sys.GetTimeZoneData(name, out int length); + if(bytes == IntPtr.Zero) + { + rawData = null; + return false; + } + + rawData = new byte[length]; + Marshal.Copy(bytes, rawData, 0, length); + return true; + } +#endif + /// /// Gets the tzfile raw data for the current 'local' time zone using the following rules. /// @@ -387,6 +446,10 @@ private static bool TryLoadTzFile(string tzFilePath, [NotNullWhen(true)] ref byt /// 2. Get the default TZ from the device /// 3. Use UTC if all else fails. /// + /// On WASI / Browser + /// 1. if TZDIR is not set, use TZ variable as id to embedded database. + /// 2. fall back to unix behavior if TZDIR is set. + /// /// On all other platforms /// 1. Read the TZ environment variable. If it is set, use it. /// 2. Look for the data in /etc/localtime. @@ -406,6 +469,11 @@ private static bool TryGetLocalTzFile([NotNullWhen(true)] out byte[]? rawData, [ { #if TARGET_IOS || TARGET_TVOS tzVariable = Interop.Sys.GetDefaultTimeZone(); +#elif TARGET_WASI || TARGET_BROWSER + if (UseEmbeddedTzDatabase) + { + return false; // use UTC + } #else return TryLoadTzFile("/etc/localtime", ref rawData, ref id) || @@ -432,6 +500,24 @@ private static bool TryGetLocalTzFile([NotNullWhen(true)] out byte[]? rawData, [ { tzFilePath = tzVariable; } + +#if TARGET_WASI || TARGET_BROWSER + if (UseEmbeddedTzDatabase) + { + // embedded database only supports relative paths + if (tzVariable[0] == '/') + { + return false; + } + if(!TryLoadEmbeddedTzFile(tzFilePath, out rawData)) + { + return false; + } + id = tzVariable; + return true; + } +#endif + return TryLoadTzFile(tzFilePath, ref rawData, ref id); } diff --git a/src/mono/mono/utils/mono-dl-wasm.c b/src/mono/mono/utils/mono-dl-wasm.c index 15189fe9db0cb..774d1ba6dd2b1 100644 --- a/src/mono/mono/utils/mono-dl-wasm.c +++ b/src/mono/mono/utils/mono-dl-wasm.c @@ -1,5 +1,6 @@ #include #include +#include #if defined (HOST_WASM) @@ -93,3 +94,45 @@ mono_dl_close_handle (MonoDl *module, MonoError *error) MONO_EMPTY_SOURCE_FILE (mono_dl_wasm); #endif + +#if defined (HOST_WASM) + +static GHashTable *name_to_blob = NULL; + +typedef struct { + const unsigned char *data; + unsigned int size; +} FileBlob; + +int +mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size) +{ + // printf("mono_wasm_add_bundled_file: %s %p %d\n", name, data, size); + if(name_to_blob == NULL) + { + name_to_blob = g_hash_table_new (g_str_hash, g_str_equal); + } + FileBlob *blob = g_new0 (FileBlob, 1); + blob->data = data; + blob->size = size; + g_hash_table_insert (name_to_blob, (gpointer) name, blob); + return 0; +} + +const unsigned char* +mono_wasm_get_bundled_file (const char *name, int* out_length) +{ + FileBlob *blob = (FileBlob *)g_hash_table_lookup (name_to_blob, name); + if (blob != NULL) + { + // printf("mono_wasm_get_bundled_file: %s %p %d \n", name, blob->data, blob->size); + *out_length = blob->size; + return blob->data; + } + + // printf("mono_wasm_get_bundled_file: %s not found \n", name); + *out_length = 0; + return NULL; +} + +#endif /* HOST_WASM */ diff --git a/src/mono/mono/utils/mono-dl.h b/src/mono/mono/utils/mono-dl.h index 40d5fff11d094..d3893ff8a7139 100644 --- a/src/mono/mono/utils/mono-dl.h +++ b/src/mono/mono/utils/mono-dl.h @@ -58,5 +58,10 @@ int mono_dl_convert_flags (int mono_flags, int native_flags); char* mono_dl_current_error_string (void); const char* mono_dl_get_system_dir (void); +#if defined (HOST_WASM) +int mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size); +const unsigned char* mono_wasm_get_bundled_file (const char *name, int* out_length); +#endif /* HOST_WASM */ + #endif /* __MONO_UTILS_DL_H__ */ diff --git a/src/mono/wasi/Wasi.Build.Tests/BuildTestBase.cs b/src/mono/wasi/Wasi.Build.Tests/BuildTestBase.cs index a66b793a08ed0..8f3857c53f182 100644 --- a/src/mono/wasi/Wasi.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasi/Wasi.Build.Tests/BuildTestBase.cs @@ -360,7 +360,6 @@ static void AssertRuntimePackPath(string buildOutput, string targetFramework) { "index.html", mainJS, - "dotnet.timezones.blat", "dotnet.wasm", "mono-config.json", "dotnet.js" diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 9fa1d9f0b9d1c..ef10347b0341f 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -1,7 +1,6 @@ - @@ -169,6 +168,8 @@ <_WasmCommonCFlags Include="-DGEN_PINVOKE=1" /> + <_WasmCommonCFlags Condition="'$(WasmSingleFileBundle)' == 'true'" Include="-DBUNDLED_ASSEMBLIES=1" /> + <_WasmCommonCFlags Include="-DINVARIANT_GLOBALIZATION=1" Condition="'$(InvariantGlobalization)' == 'true'"/> @@ -344,40 +345,40 @@ + + - <_GetBundledFileSourcePath>$(_WasmIntermediateOutputPath)dotnet_wasi_getbundledfile.c + <_WasmAssembliesBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_assemblies.o - - - + - - $(_WasmIntermediateOutputPath)%(WasmBundleFilesWithHashes.Filename)%(WasmBundleFilesWithHashes.Extension).$([System.String]::Copy(%(WasmBundleFilesWithHashes.FileHash)).Substring(0, 8)).o - + + $(_WasmIntermediateOutputPath)%(WasmBundleAssembliesWithHashes.Filename)%(WasmBundleAssembliesWithHashes.Extension).$([System.String]::Copy(%(WasmBundleAssembliesWithHashes.FileHash)).Substring(0, 8)).o + - - - - - - + + + - <_WasiObjectFilesForBundle Include="$(_GetBundledFileSourcePath)" /> - <_WasiObjectFilesForBundle Include="%(WasmBundleFilesWithHashes.ObjectFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmAssembliesBundleObjectFile)" /> + <_WasiObjectFilesForBundle Include="%(WasmBundleAssembliesWithHashes.DestinationFile)" /> - + + diff --git a/src/mono/wasi/build/WasiApp.targets b/src/mono/wasi/build/WasiApp.targets index 4292ed3df94b4..6085aa2e61ad7 100644 --- a/src/mono/wasi/build/WasiApp.targets +++ b/src/mono/wasi/build/WasiApp.targets @@ -324,7 +324,6 @@ - diff --git a/src/mono/wasi/build/WasiSdk.Defaults.props b/src/mono/wasi/build/WasiSdk.Defaults.props index 67e2e51380200..baa50e602670f 100644 --- a/src/mono/wasi/build/WasiSdk.Defaults.props +++ b/src/mono/wasi/build/WasiSdk.Defaults.props @@ -4,5 +4,5 @@ $([MSBuild]::NormalizeDirectory($(WasiSdkRoot), 'share', 'wasi-sysroot')) $(WasiSdkRoot)\bin\clang $(WasiClang).exe - + diff --git a/src/mono/wasi/runtime/CMakeLists.txt b/src/mono/wasi/runtime/CMakeLists.txt index 750e47d275f76..cb9c20dc5308d 100644 --- a/src/mono/wasi/runtime/CMakeLists.txt +++ b/src/mono/wasi/runtime/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries(dotnet ${MONO_ARTIFACTS_DIR}/libmono-ee-interp.a ${MONO_ARTIFACTS_DIR}/libmonosgen-2.0.a ${MONO_ARTIFACTS_DIR}/libmono-icall-table.a + ${NATIVE_BIN_DIR}/wasm-bundled-timezones.a ${NATIVE_BIN_DIR}/libSystem.Native.a ${NATIVE_BIN_DIR}/libSystem.IO.Compression.Native.a ) diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index b6b24bd94f2fa..533b0afd4498f 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -62,8 +62,11 @@ int32_t monoeg_g_hasenv(const char *variable); void mono_free (void*); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); -extern const char* dotnet_wasi_getbundledfile(const char* name, int* out_length); -extern void dotnet_wasi_registerbundledassemblies(); +extern void mono_wasm_register_timezones_bundle(); +#ifdef BUNDLED_ASSEMBLIES +extern void mono_wasm_register_assemblies_bundle(); +#endif + extern const char* dotnet_wasi_getentrypointassemblyname(); int32_t mono_wasi_load_icu_data(const void* pData); void load_icu_data (void); @@ -427,6 +430,10 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); + mono_wasm_register_timezones_bundle(); +#ifdef BUNDLED_ASSEMBLIES + mono_wasm_register_assemblies_bundle(); +#endif mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); diff --git a/src/mono/wasi/runtime/main.c b/src/mono/wasi/runtime/main.c index 3b0e5a322864d..ff0a25e414406 100644 --- a/src/mono/wasi/runtime/main.c +++ b/src/mono/wasi/runtime/main.c @@ -5,18 +5,12 @@ // This symbol's implementation is generated during the build const char* dotnet_wasi_getentrypointassemblyname(); -// These are generated by EmitWasmBundleObjectFile -const char* dotnet_wasi_getbundledfile(const char* name, int* out_length); -void dotnet_wasi_registerbundledassemblies(); - #ifdef WASI_AFTER_RUNTIME_LOADED_DECLARATIONS // This is supplied from the MSBuild itemgroup @(WasiAfterRuntimeLoaded) WASI_AFTER_RUNTIME_LOADED_DECLARATIONS #endif int main(int argc, char * argv[]) { - // generated during the build - dotnet_wasi_registerbundledassemblies(); #ifdef WASI_AFTER_RUNTIME_LOADED_CALLS // This is supplied from the MSBuild itemgroup @(WasiAfterRuntimeLoaded) diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index a7f58f0aac4f6..80f3379462146 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -10,11 +10,16 @@ $([MSBuild]::NormalizeDirectory('$(PkgMicrosoft_NETCore_Runtime_ICU_Transport)', 'runtimes', 'wasi-wasm-threads', 'native', 'lib')) false false - $(ArtifactsObjDir)wasm + $(ArtifactsObjDir)wasi <_WasiDefaultsRspPath>$(NativeBinDir)src\wasi-default.rsp <_WasiCompileRspPath>$(NativeBinDir)src\wasi-compile.rsp <_WasiLinkRspPath>$(NativeBinDir)src\wasi-link.rsp false + $(WASI_SDK_PATH) + $(WasiSdkRoot)\bin\clang + $(WasiClang).exe + $(WasiSdkRoot)\bin\llvm-ar + $(WasiLLVMAr).exe @@ -28,23 +33,12 @@ - - - - $(NativeBinDir)dotnet.timezones.blat - - - - - $(WasmObjDir)\pinvoke-table.h - $(WasmObjDir)\wasm_m2n_invoke.g.h + $(WasiObjDir)\pinvoke-table.h + $(WasiObjDir)\wasm_m2n_invoke.g.h @@ -60,7 +54,7 @@ - + + + + + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) + <_WasmTimezonesBundleObjectFile>$(WasiObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundleArchive>$(WasiObjDir)\wasm-bundled-timezones.a + <_WasmTimezonesArchiveRsp>$(WasiObjDir)\wasm-bundled-timezones-archive.rsp + + + <_WasmTimezonesInternal Include="$(_WasmTimezonesPath)\**\*.*" WasmRole="Timezone"/> + + + + + + <_WasmBundleTimezonesWithHashes Update="@(_WasmBundleTimezonesWithHashes)"> + $(WasiObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).o + /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) + + + + + + + + + + + + + + <_WasmArchivedTimezones Include="$(_WasmTimezonesBundleArchive)" /> + + + + + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> + <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> + + + @@ -143,7 +188,7 @@ + DependsOnTargets="GenerateWasiPropsAndRspFiles;GenerateManagedToNative;GenerateTimezonesArchive"> - $(ArtifactsObjDir)wasm/pinvoke-table.h - $(ArtifactsObjDir)wasm/wasm_m2n_invoke.g.h + $(WasiObjDir)\pinvoke-table.h + $(WasiObjDir)\wasm_m2n_invoke.g.h $([MSBuild]::EnsureTrailingSlash('$(WASI_SDK_PATH)').Replace('\', '/')) -g -Os -DDEBUG=1 -DENABLE_AOT_PROFILER=1 @@ -196,9 +241,9 @@ <_FilesToCopy Include="$(MSBuildThisFileDirectory)runtime/stubs.c" DestinationFolder="$(NativeBinDir)src" /> <_FilesToCopy Include="$(MSBuildThisFileDirectory)runtime/synthetic-pthread.c" DestinationFolder="$(NativeBinDir)src" /> - <_FilesToCopy Include="$(MonoProjectRoot)wasi\mono-include\driver.h" DestinationFolder="$(NativeBinDir)include\wasm" /> - <_FilesToCopy Include="$(MonoProjectRoot)wasi\mono-include\pinvoke.h" DestinationFolder="$(NativeBinDir)include\wasm" /> - <_FilesToCopy Include="$(MonoProjectRoot)wasm\runtime\gc-common.h" DestinationFolder="$(NativeBinDir)include\wasm" /> + <_FilesToCopy Include="$(MonoProjectRoot)wasi\mono-include\driver.h" DestinationFolder="$(NativeBinDir)include\wasm" /> + <_FilesToCopy Include="$(MonoProjectRoot)wasi\mono-include\pinvoke.h" DestinationFolder="$(NativeBinDir)include\wasm" /> + <_FilesToCopy Include="$(MonoProjectRoot)wasm\runtime\gc-common.h" DestinationFolder="$(NativeBinDir)include\wasm" /> @@ -227,12 +273,11 @@ - - diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 39e02f4c79969..ee464a7364c52 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -649,7 +649,6 @@ static void AssertRuntimePackPath(string buildOutput, string targetFramework) { "index.html", mainJS, - "dotnet.timezones.blat", "dotnet.wasm", "mono-config.json", "dotnet.js" diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index f9e80e23a6268..95b6c5f9d86b3 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -334,7 +334,6 @@ Condition="'$(WasmEmitSymbolMap)' == 'true' and '$(_HasDotnetJsSymbols)' != 'true' and Exists('$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.js.symbols')" /> - diff --git a/src/mono/wasm/host/WebServerStartup.cs b/src/mono/wasm/host/WebServerStartup.cs index 9285c83db6e6b..07fdec475f205 100644 --- a/src/mono/wasm/host/WebServerStartup.cs +++ b/src/mono/wasm/host/WebServerStartup.cs @@ -82,7 +82,7 @@ static int GetNextRandomExcept(Range range, params int[] except) provider.Mappings[".cjs"] = "text/javascript"; provider.Mappings[".mjs"] = "text/javascript"; - foreach (string extn in new string[] { ".dll", ".pdb", ".dat", ".blat", ".webcil" }) + foreach (string extn in new string[] { ".dll", ".pdb", ".dat", ".webcil" }) { provider.Mappings[extn] = "application/octet-stream"; } diff --git a/src/mono/wasm/runtime/CMakeLists.txt b/src/mono/wasm/runtime/CMakeLists.txt index 26c3c7e25f8c3..98ed82a16229c 100644 --- a/src/mono/wasm/runtime/CMakeLists.txt +++ b/src/mono/wasm/runtime/CMakeLists.txt @@ -27,6 +27,7 @@ target_link_libraries(dotnet ${MONO_ARTIFACTS_DIR}/libmono-wasm-eh-js.a ${MONO_ARTIFACTS_DIR}/libmono-profiler-aot.a ${MONO_ARTIFACTS_DIR}/libmono-profiler-browser.a + ${NATIVE_BIN_DIR}/wasm-bundled-timezones.a ${NATIVE_BIN_DIR}/libSystem.Native.a ${NATIVE_BIN_DIR}/libSystem.IO.Compression.Native.a) diff --git a/src/mono/wasm/runtime/assets.ts b/src/mono/wasm/runtime/assets.ts index c485d0d8d94ce..38544d0d94894 100644 --- a/src/mono/wasm/runtime/assets.ts +++ b/src/mono/wasm/runtime/assets.ts @@ -513,11 +513,6 @@ export function mono_wasm_load_data_archive(data: Uint8Array, prefix: string): b data = data.slice(manifestSize + 8); // Create the folder structure - // /usr/share/zoneinfo - // /usr/share/zoneinfo/Africa - // /usr/share/zoneinfo/Asia - // .. - const folders = new Set(); manifest.filter(m => { const file = m[0]; diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index 9b145963b0d4c..0b8d75361ce2f 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -68,6 +68,7 @@ void mono_free (void*); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); char *mono_method_full_name (MonoMethod *method, int signature); +extern void mono_wasm_register_timezones_bundle(); static void mono_wasm_init_finalizer_thread (void); @@ -537,6 +538,7 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); + mono_wasm_register_timezones_bundle(); mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); diff --git a/src/mono/wasm/runtime/startup.ts b/src/mono/wasm/runtime/startup.ts index 5c96738ba76bd..8f5f30a4b1f2e 100644 --- a/src/mono/wasm/runtime/startup.ts +++ b/src/mono/wasm/runtime/startup.ts @@ -484,11 +484,15 @@ async function instantiate_wasm_module( async function _apply_configuration_from_args() { try { const tz = Intl.DateTimeFormat().resolvedOptions().timeZone; - mono_wasm_setenv("TZ", tz || "UTC"); + if (tz) mono_wasm_setenv("TZ", tz); } catch { - mono_wasm_setenv("TZ", "UTC"); + console.info("MONO_WASM: failed to detect timezone, will fallback to UTC"); } + // create /usr/share folder which is SpecialFolder.CommonApplicationData + Module["FS_createPath"]("/", "usr", true, true); + Module["FS_createPath"]("/", "usr/share", true, true); + for (const k in config.environmentVariables) { const v = config.environmentVariables![k]; if (typeof (v) === "string") diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index ff6737584da3c..3f4d2d2e2e03e 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -34,6 +34,8 @@ <_EmccLinkRspPath>$(NativeBinDir)src\emcc-link.rsp <_EmccLinkUndefinedSymbolsFile>$(WasmObjDir)src\symbols.undefined false + $(EMSDK_PATH)\upstream\bin\llvm-ar + $(EmSdkLLVMAr).exe @@ -47,17 +49,6 @@ - - - - $(NativeBinDir)dotnet.timezones.blat - - - - @@ -89,6 +80,82 @@ + + + + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) + <_WasmTimezonesBundleSourceFile>$(WasmObjDir)\wasm-bundled-timezones.c + <_WasmTimezonesBundleObjectFile>$(WasmObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundleArchive>$(WasmObjDir)\wasm-bundled-timezones.a + <_WasmTimezonesSourcesRsp>$(WasmObjDir)\wasm-bundled-timezones-sources.rsp + <_WasmTimezonesArchiveRsp>$(WasmObjDir)\wasm-bundled-timezones-archive.rsp + + + <_WasmTimezonesInternal Include="$(_WasmTimezonesPath)\**\*.*" WasmRole="Timezone"/> + + + + + + <_WasmBundleTimezonesWithHashes Update="@(_WasmBundleTimezonesWithHashes)"> + $(WasmObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).c + $(WasmObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).o + /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) + + + + + + + <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" /> + <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundleSourceFile)).Replace('\','/'))" /> + + + + + + + + + + + + + + <_WasmArchivedTimezones Include="$(WasmObjDir)\wasm-bundled-timezones.a" /> + + + + + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.c" /> + <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleSourceFile)" /> + <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> + <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.ObjectFile)" /> + + + + @@ -249,7 +316,7 @@ + DependsOnTargets="GenerateEmccPropsAndRspFiles;GenerateManagedToNative;GenerateTimezonesArchive;InstallNpmPackages;BuildWithRollup"> @@ -347,8 +415,7 @@ $(NativeBinDir)dotnet.d.ts; $(NativeBinDir)dotnet-legacy.d.ts; $(NativeBinDir)package.json; - $(NativeBinDir)dotnet.wasm; - $(NativeBinDir)dotnet.timezones.blat" + $(NativeBinDir)dotnet.wasm" DestinationFolder="$(MicrosoftNetCoreAppRuntimePackNativeDir)" SkipUnchangedFiles="true" /> @@ -361,7 +428,7 @@ DestinationFolder="$(MicrosoftNetCoreAppRuntimePackNativeDir)" SkipUnchangedFiles="true" /> - diff --git a/src/native/libs/System.Native/entrypoints.c b/src/native/libs/System.Native/entrypoints.c index c4faf76ec7dac..6b66bd8eeb40f 100644 --- a/src/native/libs/System.Native/entrypoints.c +++ b/src/native/libs/System.Native/entrypoints.c @@ -48,6 +48,7 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_GetSignalForBreak) DllImportEntry(SystemNative_SetSignalForBreak) DllImportEntry(SystemNative_GetSystemTimeAsTicks) + DllImportEntry(SystemNative_GetTimeZoneData) DllImportEntry(SystemNative_ConvertErrorPlatformToPal) DllImportEntry(SystemNative_ConvertErrorPalToPlatform) DllImportEntry(SystemNative_StrErrorR) diff --git a/src/native/libs/System.Native/pal_datetime.c b/src/native/libs/System.Native/pal_datetime.c index 73c0a2583e450..61b0dba23a919 100644 --- a/src/native/libs/System.Native/pal_datetime.c +++ b/src/native/libs/System.Native/pal_datetime.c @@ -3,6 +3,7 @@ #include "pal_config.h" #include "pal_datetime.h" +#include "pal_utilities.h" #include #include #include @@ -19,6 +20,10 @@ static const int64_t NANOSECONDS_PER_TICK = 100; static const int64_t TICKS_PER_MICROSECOND = 10; /* 1000 / 100 */ #endif +#if defined(TARGET_WASI) || defined(TARGET_BROWSER) +extern const unsigned char* mono_wasm_get_bundled_file(const char* name, int* out_length); +#endif + // // SystemNative_GetSystemTimeAsTicks return the system time as ticks (100 nanoseconds) // since 00:00 01 January 1970 UTC (Unix epoch) @@ -56,3 +61,17 @@ char* SystemNative_GetDefaultTimeZone(void) } } #endif + +const char* SystemNative_GetTimeZoneData(const char* name, int* length) +{ + assert(name != NULL); + assert(length != NULL); +#if defined(TARGET_WASI) || defined(TARGET_BROWSER) + return (const char*) mono_wasm_get_bundled_file(name, length); +#else + assert_msg(false, "Not supported on this platform", 0); + (void)name; // unused + (void)length; // unused + return NULL; +#endif +} diff --git a/src/native/libs/System.Native/pal_datetime.h b/src/native/libs/System.Native/pal_datetime.h index bc59762a066b0..b6da4a89cc327 100644 --- a/src/native/libs/System.Native/pal_datetime.h +++ b/src/native/libs/System.Native/pal_datetime.h @@ -11,3 +11,5 @@ PALEXPORT int64_t SystemNative_GetSystemTimeAsTicks(void); #if defined(TARGET_ANDROID) || defined(__APPLE__) PALEXPORT char* SystemNative_GetDefaultTimeZone(void); #endif + +PALEXPORT const char* SystemNative_GetTimeZoneData(const char* name, int* length); diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 8ed6c2c13711e..91e5b77c1e198 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -113,7 +113,8 @@ static string CreateTemporaryBatchFile(string command) bool silent = true, bool logStdErrAsMessage = false, MessageImportance debugMessageImportance=MessageImportance.High, - string? label=null) + string? label=null, + Action? inputProvider = null) { string msgPrefix = label == null ? string.Empty : $"[{label}] "; logger.LogMessage(debugMessageImportance, $"{msgPrefix}Running: {path} {args}"); @@ -125,6 +126,7 @@ static string CreateTemporaryBatchFile(string command) CreateNoWindow = true, RedirectStandardError = true, RedirectStandardOutput = true, + RedirectStandardInput = inputProvider != null, Arguments = args, }; @@ -181,6 +183,7 @@ static string CreateTemporaryBatchFile(string command) }; process.BeginOutputReadLine(); process.BeginErrorReadLine(); + inputProvider?.Invoke(process.StandardInput.BaseStream); process.WaitForExit(); logger.LogMessage(debugMessageImportance, $"{msgPrefix}Exit code: {process.ExitCode}"); diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs b/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs new file mode 100644 index 0000000000000..7286e088a65fb --- /dev/null +++ b/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs @@ -0,0 +1,240 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; + +namespace Microsoft.WebAssembly.Build.Tasks; + +public abstract class EmitWasmBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource BuildTaskCancelled { get; } = new(); + + /// Must have DestinationFile metadata, which is the output filename + /// Could have RegisteredName, otherwise it would be the filename. + /// RegisteredName should be prefixed with namespace in form of unix like path. For example: "/usr/share/zoneinfo/" + [Required] + public ITaskItem[] FilesToBundle { get; set; } = default!; + + [Required] + public string BundleName { get; set; } = default!; + + [Required] + public string BundleFile { get; set; } = default!; + + [Required] + public string RegistrationCallbackFunctionName { get; set; } = default!; + + public override bool Execute() + { + // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore + // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. + var filesToBundleByDestinationFileName = FilesToBundle.GroupBy(f => f.GetMetadata("DestinationFile")).ToList(); + + // We're handling the incrementalism within this task, because it needs to be based on file content hashes + // and not on timetamps. The output filenames contain a content hash, so if any such file already exists on + // disk with that name, we know it must be up-to-date. + var remainingDestinationFilesToBundle = filesToBundleByDestinationFileName.Where(g => !File.Exists(g.Key)).ToArray(); + + // If you're only touching the leaf project, we don't really need to tell you that. + // But if there's more work to do it's valuable to show progress. + var verbose = remainingDestinationFilesToBundle.Length > 1; + var verboseCount = 0; + + var filesToBundleByRegisteredName = FilesToBundle.GroupBy(file => { + var registeredName = file.GetMetadata("RegisteredName"); + if(string.IsNullOrEmpty(registeredName)) + { + registeredName = Path.GetFileName(file.ItemSpec); + } + return registeredName; + }).ToList(); + + var files = filesToBundleByRegisteredName.Select(group => { + var registeredFile = group.First(); + var outputFile = registeredFile.GetMetadata("DestinationFile"); + var registeredName = group.Key; + var symbolName = ToSafeSymbolName(outputFile); + return (registeredName, symbolName); + }).ToList(); + + Log.LogMessage(MessageImportance.Low, "Bundling {numFiles} files for {bundleName}", files.Count, BundleName); + + if (remainingDestinationFilesToBundle.Length > 0) + { + int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + + Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => + { + var group = remainingDestinationFilesToBundle[i]; + + // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, + // since we know each group's ITaskItems all contain the same binary data + var contentSourceFile = group.First(); + + var outputFile = group.Key; + var inputFile = contentSourceFile.ItemSpec; + if (verbose) + { + var registeredName = contentSourceFile.GetMetadata("RegisteredName"); + if(string.IsNullOrEmpty(registeredName)) + { + registeredName = Path.GetFileName(inputFile); + } + var count = Interlocked.Increment(ref verboseCount); + Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); + } + + Log.LogMessage(MessageImportance.Low, "Bundling {0} as {1}", inputFile, outputFile); + var symbolName = ToSafeSymbolName(outputFile); + if (!Emit(outputFile, (codeStream) => { + using var inputStream = File.OpenRead(inputFile); + BundleFileToCSource(symbolName, inputStream, codeStream); + })) + { + state.Stop(); + } + }); + } + + return Emit(BundleFile, (inputStream) => + { + using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); + GenerateRegisteredBundledObjects($"mono_wasm_register_{BundleName}_bundle", RegistrationCallbackFunctionName, files, outputUtf8Writer); + }) && !Log.HasLoggedErrors; + } + + public void Cancel() + { + BuildTaskCancelled.Cancel(); + } + + #region Helpers + + private static readonly Encoding Utf8NoBom = new UTF8Encoding(false); + private static readonly byte[] HexToUtf8Lookup = InitLookupTable(); + private static readonly byte[] NewLineAndIndentation = new[] { (byte)0x0a, (byte)0x20, (byte)0x20 }; + + private static byte[] InitLookupTable() + { + // Every 6 bytes in this array represents the output for a different input byte value. + // For example, the input byte 0x1a (26 decimal) corresponds to bytes 156-161 (26*6=156), + // whose values will be ['0', 'x', '1', 'a', ',', ' '], which is the UTF-8 representation + // for "0x1a, ". This is just a faster alternative to calling .ToString("x2") on every + // byte of the input file and then pushing that string through UTF8Encoding. + var lookup = new byte[256 * 6]; + for (int i = 0; i < 256; i++) + { + string byteAsHex = i.ToString("x2"); + char highOrderChar = BitConverter.IsLittleEndian ? byteAsHex[0] : byteAsHex[1]; + char lowOrderChar = BitConverter.IsLittleEndian ? byteAsHex[1] : byteAsHex[0]; + lookup[i * 6 + 0] = (byte)'0'; + lookup[i * 6 + 1] = (byte)'x'; + lookup[i * 6 + 2] = (byte)highOrderChar; + lookup[i * 6 + 3] = (byte)lowOrderChar; + lookup[i * 6 + 4] = (byte)','; + lookup[i * 6 + 5] = (byte)' '; + } + + return lookup; + } + + public abstract bool Emit(string destinationFile, Action inputProvider); + + public static void GenerateRegisteredBundledObjects(string newFunctionName, string callbackFunctionName, ICollection<(string registeredName, string symbol)> files, StreamWriter outputUtf8Writer) + { + outputUtf8Writer.WriteLine($"int {callbackFunctionName}(const char* name, const unsigned char* data, unsigned int size);"); + outputUtf8Writer.WriteLine(); + + foreach (var tuple in files) + { + outputUtf8Writer.WriteLine($"extern const unsigned char {tuple.symbol}[];"); + outputUtf8Writer.WriteLine($"extern const int {tuple.symbol}_len;"); + } + + outputUtf8Writer.WriteLine(); + outputUtf8Writer.WriteLine($"void {newFunctionName}() {{"); + + foreach (var tuple in files) + { + outputUtf8Writer.WriteLine($" {callbackFunctionName} (\"{tuple.registeredName}\", {tuple.symbol}, {tuple.symbol}_len);"); + } + + outputUtf8Writer.WriteLine("}"); + } + + private static void BundleFileToCSource(string symbolName, FileStream inputStream, Stream outputStream) + { + // Emits a C source file in the same format as "xxd --include". Example: + // + // unsigned char Some_File_dll[] = { + // 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x0a + // }; + // unsigned int Some_File_dll_len = 6; + + var buf = new byte[4096]; + int bytesRead; + var generatedArrayLength = 0; + var bytesEmitted = 0; + + using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); + + outputUtf8Writer.Write($"unsigned char {symbolName}[] = {{"); + outputUtf8Writer.Flush(); + while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) + { + for (var i = 0; i < bytesRead; i++) + { + if (bytesEmitted++ % 12 == 0) + { + outputStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); + } + + var byteValue = buf[i]; + outputStream.Write(HexToUtf8Lookup, byteValue * 6, 6); + } + + generatedArrayLength += bytesRead; + } + + outputUtf8Writer.WriteLine("0\n};"); + outputUtf8Writer.WriteLine($"unsigned int {symbolName}_len = {generatedArrayLength};"); + outputUtf8Writer.Flush(); + outputStream.Flush(); + } + + private static string ToSafeSymbolName(string destinationFileName) + { + // Since destinationFileName includes a content hash, we can safely strip off the directory name + // as the filename is always unique enough. This avoid disclosing information about the build + // file structure in the resulting symbols. + var filename = Path.GetFileName(destinationFileName); + + // Equivalent to the logic from "xxd --include" + var sb = new StringBuilder(); + foreach (var c in filename) + { + sb.Append(IsAlphanumeric(c) ? c : '_'); + } + + return sb.ToString(); + } + + // Equivalent to "isalnum" + private static bool IsAlphanumeric(char c) => c + is (>= 'a' and <= 'z') + or (>= 'A' and <= 'Z') + or (>= '0' and <= '9'); + + #endregion + +} diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs b/src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs new file mode 100644 index 0000000000000..48c1f509768cd --- /dev/null +++ b/src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; +using Microsoft.Build.Framework; + +namespace Microsoft.WebAssembly.Build.Tasks; + +public class EmitWasmBundleObjectFiles : EmitWasmBundleBase +{ + [Required] + public string ClangExecutable { get; set; } = default!; + + public override bool Execute() + { + if (!File.Exists(ClangExecutable)) + { + Log.LogError($"Cannot find {nameof(ClangExecutable)}={ClangExecutable}"); + return false; + } + + return base.Execute(); + } + + public override bool Emit(string destinationFile, Action inputProvider) + { + if (Path.GetDirectoryName(destinationFile) is string destDir && !string.IsNullOrEmpty(destDir)) + Directory.CreateDirectory(destDir); + + (int exitCode, string output) = Utils.TryRunProcess(Log, + ClangExecutable!, + args: $"-xc -o \"{destinationFile}\" -c -", + envVars: null, workingDir: null, silent: true, logStdErrAsMessage: false, + debugMessageImportance: MessageImportance.Low, label: null, + inputProvider); + if (exitCode != 0) + { + Log.LogError($"Failed to compile with exit code {exitCode}{Environment.NewLine}Output: {output}"); + } + return exitCode == 0; + } + +} diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs b/src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs new file mode 100644 index 0000000000000..b57a8f4627a82 --- /dev/null +++ b/src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using Microsoft.Build.Framework; + +namespace Microsoft.WebAssembly.Build.Tasks; + +// It would be ideal that this Task would always produce object files as EmitWasmBundleObjectFiles does. +// EmitWasmBundleObjectFiles could do it with clang by streaming code directly to clang input stream. +// For emcc it's not possible, so we need to write the code to disk first and then compile it in MSBuild. +public class EmitWasmBundleSourceFiles : EmitWasmBundleBase +{ + public override bool Emit(string destinationFile, Action inputProvider) + { + using (var fileStream = File.Create(destinationFile)) + { + inputProvider(fileStream); + } + + return true; + } +} diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 89e3322cac294..085efe4d7b93f 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -322,7 +322,6 @@ protected override bool ExecuteInternal() } - config.Assets.Add(new VfsEntry ("dotnet.timezones.blat") { VirtualPath = "/usr/share/zoneinfo/"}); config.Assets.Add(new WasmEntry ("dotnet.wasm") ); if (IncludeThreadsWorker) config.Assets.Add(new ThreadsWorkerEntry ("dotnet.worker.js") ); diff --git a/src/tasks/WasmAppBuilder/wasi/EmitWasmBundleObjectFile.cs b/src/tasks/WasmAppBuilder/wasi/EmitWasmBundleObjectFile.cs deleted file mode 100644 index 2695e526e211e..0000000000000 --- a/src/tasks/WasmAppBuilder/wasi/EmitWasmBundleObjectFile.cs +++ /dev/null @@ -1,289 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Build.Framework; - -namespace Microsoft.WebAssembly.Build.Tasks; - -public class EmitWasmBundleObjectFile : Microsoft.Build.Utilities.Task, ICancelableTask -{ - private static readonly Encoding Utf8NoBom = new UTF8Encoding(false); - private static readonly byte[] HexToUtf8Lookup = InitLookupTable(); - private static readonly byte[] NewLineAndIndentation = new[] { (byte)0x0a, (byte)0x20, (byte)0x20 }; - private CancellationTokenSource BuildTaskCancelled { get; } = new(); - - // We only want to emit a single copy of the data for a given content hash, but we have to track all the - // different filenames that may be referencing that content - private ICollection> _filesToBundleByObjectFileName = default!; - - [Required] - public ITaskItem[] FilesToBundle { get; set; } = default!; - - [Required] - public string ClangExecutable { get; set; } = default!; - - [Output] - public string? BundleApiSourceCode { get; set; } - - private static byte[] InitLookupTable() - { - // Every 6 bytes in this array represents the output for a different input byte value. - // For example, the input byte 0x1a (26 decimal) corresponds to bytes 156-161 (26*6=156), - // whose values will be ['0', 'x', '1', 'a', ',', ' '], which is the UTF-8 representation - // for "0x1a, ". This is just a faster alternative to calling .ToString("x2") on every - // byte of the input file and then pushing that string through UTF8Encoding. - var lookup = new byte[256 * 6]; - for (int i = 0; i < 256; i++) - { - string byteAsHex = i.ToString("x2"); - char highOrderChar = BitConverter.IsLittleEndian ? byteAsHex[0] : byteAsHex[1]; - char lowOrderChar = BitConverter.IsLittleEndian ? byteAsHex[1] : byteAsHex[0]; - lookup[i * 6 + 0] = (byte)'0'; - lookup[i * 6 + 1] = (byte)'x'; - lookup[i * 6 + 2] = (byte)highOrderChar; - lookup[i * 6 + 3] = (byte)lowOrderChar; - lookup[i * 6 + 4] = (byte)','; - lookup[i * 6 + 5] = (byte)' '; - } - - return lookup; - } - - public override bool Execute() - { - if (!File.Exists(ClangExecutable)) - { - Log.LogError($"Cannot find {nameof(ClangExecutable)}={ClangExecutable}"); - return false; - } - - // The ObjectFile (output filename) already includes a content hash. Grouping by this filename therefore - // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. - _filesToBundleByObjectFileName = FilesToBundle.GroupBy(f => f.GetMetadata("ObjectFile")).ToList(); - - // We're handling the incrementalism within this task, because it needs to be based on file content hashes - // and not on timetamps. The output filenames contain a content hash, so if any such file already exists on - // disk with that name, we know it must be up-to-date. - var remainingObjectFilesToBundle = _filesToBundleByObjectFileName.Where(g => !File.Exists(g.Key)).ToArray(); - - // If you're only touching the leaf project, we don't really need to tell you that. - // But if there's more work to do it's valuable to show progress. - var verbose = remainingObjectFilesToBundle.Length > 1; - var verboseCount = 0; - - if (remainingObjectFilesToBundle.Length > 0) - { - int allowedParallelism = Math.Max(Math.Min(remainingObjectFilesToBundle.Length, Environment.ProcessorCount), 1); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); - - Parallel.For(0, remainingObjectFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => - { - var objectFile = remainingObjectFilesToBundle[i]; - - // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, - // since we know each group's ITaskItems all contain the same binary data - var contentSourceFile = objectFile.First(); - - var outputFile = objectFile.Key; - if (verbose) - { - var count = Interlocked.Increment(ref verboseCount); - Log.LogMessage(MessageImportance.High, "{0}/{1} Bundling {2}...", count, remainingObjectFilesToBundle.Length, Path.GetFileName(contentSourceFile.ItemSpec)); - } - - if (!EmitObjectFile(contentSourceFile, outputFile)) - state.Stop(); - }); - } - - BundleApiSourceCode = GetBundleFileApiSource(_filesToBundleByObjectFileName); - - return !Log.HasLoggedErrors; - } - - private bool EmitObjectFile(ITaskItem fileToBundle, string destinationObjectFile) - { - Log.LogMessage(MessageImportance.Low, "Bundling {0} as {1}", fileToBundle.ItemSpec, destinationObjectFile); - - if (Path.GetDirectoryName(destinationObjectFile) is string destDir && !string.IsNullOrEmpty(destDir)) - Directory.CreateDirectory(destDir); - - object syncObj = new(); - StringBuilder outputBuilder = new(); - var clangProcess = Process.Start(new ProcessStartInfo - { - FileName = ClangExecutable, - Arguments = $"-xc -o \"{destinationObjectFile}\" -c -", - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - })!; - - clangProcess.ErrorDataReceived += (sender, e) => - { - lock (syncObj) - { - if (!string.IsNullOrEmpty(e.Data)) - outputBuilder.AppendLine(e.Data); - } - }; - clangProcess.OutputDataReceived += (sender, e) => - { - lock (syncObj) - { - if (!string.IsNullOrEmpty(e.Data)) - outputBuilder.AppendLine(e.Data); - } - }; - clangProcess.BeginOutputReadLine(); - clangProcess.BeginErrorReadLine(); - - try - { - BundleFileToCSource(destinationObjectFile, fileToBundle, clangProcess.StandardInput.BaseStream); - clangProcess.WaitForExit(); - return true; - } - catch (IOException ioex) - { - Log.LogError($"Failed to compile because {ioex.Message}{Environment.NewLine}Output: {outputBuilder}"); - return false; - } - } - - private static string GetBundleFileApiSource(ICollection> bundledFilesByObjectFileName) - { - // Emit an object file that uses all the bundle file symbols and supplies an API - // for getting the bundled file data at runtime - var result = new StringBuilder(); - - result.AppendLine("#include "); - result.AppendLine(); - result.AppendLine("int mono_wasm_add_assembly(const char* name, const unsigned char* data, unsigned int size);"); - result.AppendLine(); - - foreach (var objectFileGroup in bundledFilesByObjectFileName) - { - var symbol = ToSafeSymbolName(objectFileGroup.Key); - result.AppendLine($"extern const unsigned char {symbol}[];"); - result.AppendLine($"extern const int {symbol}_len;"); - } - - result.AppendLine(); - result.AppendLine("const unsigned char* dotnet_wasi_getbundledfile(const char* name, int* out_length) {"); - - // TODO: Instead of a naive O(N) search through all bundled files, consider putting them in a - // hashtable or at least generating a series of comparisons equivalent to a binary search - - foreach (var objectFileGroup in bundledFilesByObjectFileName) - { - foreach (var file in objectFileGroup.Where(f => !string.Equals(f.GetMetadata("WasmRole"), "assembly", StringComparison.OrdinalIgnoreCase))) - { - var symbol = ToSafeSymbolName(objectFileGroup.Key); - result.AppendLine($" if (!strcmp (name, \"{file.ItemSpec.Replace("\\", "/")}\")) {{"); - result.AppendLine($" *out_length = {symbol}_len;"); - result.AppendLine($" return {symbol};"); - result.AppendLine(" }"); - result.AppendLine(); - } - } - - result.AppendLine(" return NULL;"); - result.AppendLine("}"); - - result.AppendLine(); - result.AppendLine("void dotnet_wasi_registerbundledassemblies() {"); - - foreach (var objectFileGroup in bundledFilesByObjectFileName) - { - foreach (var file in objectFileGroup.Where(f => string.Equals(f.GetMetadata("WasmRole"), "assembly", StringComparison.OrdinalIgnoreCase))) - { - var symbol = ToSafeSymbolName(objectFileGroup.Key); - result.AppendLine($" mono_wasm_add_assembly (\"{Path.GetFileName(file.ItemSpec)}\", {symbol}, {symbol}_len);"); - } - } - - result.AppendLine("}"); - - return result.ToString(); - } - - private static void BundleFileToCSource(string objectFileName, ITaskItem fileToBundle, Stream outputStream) - { - // Emits a C source file in the same format as "xxd --include". Example: - // - // unsigned char Some_File_dll[] = { - // 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x0a - // }; - // unsigned int Some_File_dll_len = 6; - - using var inputStream = File.OpenRead(fileToBundle.ItemSpec); - using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); - - var symbolName = ToSafeSymbolName(objectFileName); - outputUtf8Writer.Write($"unsigned char {symbolName}[] = {{"); - outputUtf8Writer.Flush(); - - var buf = new byte[4096]; - var bytesRead = 0; - var generatedArrayLength = 0; - var bytesEmitted = 0; - while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) - { - for (var i = 0; i < bytesRead; i++) - { - if (bytesEmitted++ % 12 == 0) - { - outputStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); - } - - var byteValue = buf[i]; - outputStream.Write(HexToUtf8Lookup, byteValue * 6, 6); - } - - generatedArrayLength += bytesRead; - } - - outputStream.Flush(); - outputUtf8Writer.WriteLine("0\n};"); - outputUtf8Writer.WriteLine($"unsigned int {symbolName}_len = {generatedArrayLength};"); - } - - private static string ToSafeSymbolName(string objectFileName) - { - // Since objectFileName includes a content hash, we can safely strip off the directory name - // as the filename is always unique enough. This avoid disclosing information about the build - // file structure in the resulting symbols. - var filename = Path.GetFileName(objectFileName); - - // Equivalent to the logic from "xxd --include" - var sb = new StringBuilder(); - foreach (var c in filename) - { - sb.Append(IsAlphanumeric(c) ? c : '_'); - } - - return sb.ToString(); - } - - // Equivalent to "isalnum" - private static bool IsAlphanumeric(char c) => c - is (>= 'a' and <= 'z') - or (>= 'A' and <= 'Z') - or (>= '0' and <= '9'); - - public void Cancel() - { - BuildTaskCancelled.Cancel(); - } -} diff --git a/src/tasks/WasmAppBuilder/wasi/WasmResolveAssemblyDependencies.cs b/src/tasks/WasmAppBuilder/wasi/WasmResolveAssemblyDependencies.cs deleted file mode 100644 index f0571c8eb3abf..0000000000000 --- a/src/tasks/WasmAppBuilder/wasi/WasmResolveAssemblyDependencies.cs +++ /dev/null @@ -1,199 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection.Metadata; -using System.Reflection.PortableExecutable; -using Microsoft.Build.Framework; -using TaskItem = Microsoft.Build.Utilities.TaskItem; - -namespace Microsoft.WebAssembly.Build.Tasks; - -/// -/// Starting from the entrypoint assembly, walks the graph of referenced assemblies using candidates from the -/// runtime pack (first priority) or application assembly list (second priority). This is a way of reducing the -/// number of bundled assemblies to the minimal set, instead of including every possible assembly from the runtime -/// pack and all framework references. -/// -public class WasmResolveAssemblyDependencies : Microsoft.Build.Utilities.Task -{ - [Required] - public string EntryPoint { get; set; } = default!; - - [Required] - public ITaskItem[] ApplicationAssemblies { get; set; } = default!; - - [Required] - public ITaskItem[] WasiRuntimePackAssemblies { get; set; } = default!; - - [Output] - public ITaskItem[]? Dependencies { get; set; } - - public override bool Execute() - { - var paths = ResolveRuntimeDependenciesCore(EntryPoint, ApplicationAssemblies, WasiRuntimePackAssemblies); - Dependencies = paths.Select(p => new TaskItem(p.Path)).ToArray(); - - return true; - } - - private static List ResolveRuntimeDependenciesCore( - string entryPointPath, - IEnumerable applicationAssemblies, - IEnumerable runtimePackAssemblies) - { - var entryAssembly = new AssemblyEntry(entryPointPath, GetAssemblyName(entryPointPath), originalTaskItem: null); - var applicationAssemblyEntries = CreateAssemblyLookup(applicationAssemblies); - var runtimePackAssemblyEntries = CreateAssemblyLookup(runtimePackAssemblies); - - var assemblyResolutionContext = new AssemblyResolutionContext( - entryAssembly, - applicationAssemblyEntries, - runtimePackAssemblyEntries); - assemblyResolutionContext.ResolveAssemblies(); - - return assemblyResolutionContext.Results; - } - - private static Dictionary CreateAssemblyLookup(IEnumerable assemblies) - { - var dictionary = new Dictionary(StringComparer.Ordinal); - foreach (var assembly in assemblies) - { - var assemblyName = GetAssemblyName(assembly.ItemSpec); - if (dictionary.TryGetValue(assemblyName, out var previous)) - { - throw new InvalidOperationException($"Multiple assemblies found with the same assembly name '{assemblyName}':" + - Environment.NewLine + string.Join(Environment.NewLine, previous, assembly.ItemSpec)); - } - dictionary[assemblyName] = new AssemblyEntry(assembly.ItemSpec, assemblyName, assembly); - } - - return dictionary; - } - - private static string GetAssemblyName(string assemblyPath) - { - // It would be more correct to return AssemblyName.GetAssemblyName(assemblyPath).Name, but that involves - // actually loading the assembly file and maybe hitting a BadImageFormatException if it's not actually - // something that can be loaded by the active .NET version (e.g., .NET Framework if this task is running - // inside VS). - // Instead we'll rely on the filename matching the assembly name. - return Path.GetFileNameWithoutExtension(assemblyPath); - } - - private sealed class AssemblyResolutionContext - { - public AssemblyResolutionContext( - AssemblyEntry entryAssembly, - Dictionary applicationAssemblies, - Dictionary runtimePackAssemblies) - { - EntryAssembly = entryAssembly; - ApplicationAssemblies = applicationAssemblies; - RuntimePackAssemblies = runtimePackAssemblies; - } - - public AssemblyEntry EntryAssembly { get; } - public Dictionary ApplicationAssemblies { get; } - public Dictionary RuntimePackAssemblies { get; } - - public List Results { get; } = new(); - - public void ResolveAssemblies() - { - var visitedAssemblies = new HashSet(); - var pendingAssemblies = new Stack(); - pendingAssemblies.Push(EntryAssembly.Name); - ResolveAssembliesCore(); - - void ResolveAssembliesCore() - { - while (pendingAssemblies.Count > 0) - { - var current = pendingAssemblies.Pop(); - if (visitedAssemblies.Add(current)) - { - // Not all references will be resolvable within the runtime pack. - // Skipping unresolved assemblies here is equivalent to passing "--skip-unresolved true" to the .NET linker. - if (Resolve(current) is AssemblyEntry resolved) - { - Results.Add(resolved); - var references = GetAssemblyReferences(resolved.Path); - foreach (var reference in references) - { - pendingAssemblies.Push(reference); - } - } - } - } - } - - AssemblyEntry? Resolve(string assemblyName) - { - if (string.Equals(assemblyName, EntryAssembly.Name, StringComparison.Ordinal)) - { - return EntryAssembly; - } - - // Resolution logic. For right now, we will prefer the runtime pack version of a given - // assembly if there is a candidate assembly and an equivalent runtime pack assembly. - if (RuntimePackAssemblies.TryGetValue(assemblyName, out var assembly) - || ApplicationAssemblies.TryGetValue(assemblyName, out assembly)) - { - return assembly; - } - - return null; - } - - static IReadOnlyList GetAssemblyReferences(string assemblyPath) - { - try - { - using var peReader = new PEReader(File.OpenRead(assemblyPath)); - if (!peReader.HasMetadata) - { - return Array.Empty(); // not a managed assembly - } - - var metadataReader = peReader.GetMetadataReader(); - - var references = new List(); - foreach (var handle in metadataReader.AssemblyReferences) - { - var reference = metadataReader.GetAssemblyReference(handle); - var referenceName = metadataReader.GetString(reference.Name); - - references.Add(referenceName); - } - - return references; - } - catch (BadImageFormatException) - { - // not a PE file, or invalid metadata - } - - return Array.Empty(); // not a managed assembly - } - } - } - - internal readonly struct AssemblyEntry - { - public AssemblyEntry(string path, string name, ITaskItem? originalTaskItem) - { - Path = path; - Name = name; - _originalTaskItem = originalTaskItem; - } - - private readonly ITaskItem? _originalTaskItem; - public string Path { get; } - public string Name { get; } - } -} diff --git a/src/tasks/WasmBuildTasks/GenerateWasmBundle.cs b/src/tasks/WasmBuildTasks/GenerateWasmBundle.cs deleted file mode 100644 index c4fc6a7849fc7..0000000000000 --- a/src/tasks/WasmBuildTasks/GenerateWasmBundle.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Buffers.Binary; -using System.Collections.Generic; -using System.IO; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Linq; -using System.Text.RegularExpressions; -using System.Net; -using System.Reflection; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -public class GenerateWasmBundle : Task -{ - [Required] - public string? InputDirectory { get; set; } - - [Required] - public string? OutputFileName { get; set; } - - private (byte[] json_bytes, MemoryStream stream) EnumerateData() - { - var indices = new List(); - var stream = new MemoryStream(); - - var directoryInfo = new DirectoryInfo(InputDirectory!); - - foreach (var entry in directoryInfo.EnumerateFiles("*", SearchOption.AllDirectories)) - { - var relativePath = Path.GetRelativePath(InputDirectory!, entry.FullName); - if (Path.DirectorySeparatorChar != '/') - relativePath = relativePath.Replace(Path.DirectorySeparatorChar, '/'); - - indices.Add(new object[] { relativePath, entry.Length }); - - using (var readStream = entry.OpenRead()) - readStream.CopyTo(stream); - } - - stream.Position = 0; - var jsonBytes = JsonSerializer.SerializeToUtf8Bytes(indices); - - return (jsonBytes, stream); - } - - public override bool Execute() - { - if (!Directory.Exists(InputDirectory)) - { - Log.LogError($"Input directory '{InputDirectory}' does not exist"); - return false; - } - - (byte[] json_bytes, MemoryStream stream) data = EnumerateData(); - - using (var file = File.Open(OutputFileName!, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite)) - { - var lengthBytes = new byte[4]; - var magicBytes = Encoding.ASCII.GetBytes("talb"); - BinaryPrimitives.WriteInt32LittleEndian(lengthBytes, data.json_bytes.Length); - file.Write(magicBytes); - file.Write(lengthBytes); - file.Write(data.json_bytes); - - data.stream.CopyTo(file); - } - - return true; - } -}