[wasi+browser] bundle timezones into .wasm (#82250)

Co-authored-by: Ankit Jain <radical@gmail.com>
dotnet · Mar 1, 2023 · 340508f · 340508f
1 parent 3d160bc
commit 340508f
Show file tree

Hide file tree

Showing 35 changed files with 720 additions and 687 deletions.
diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets
@@ -192,7 +192,6 @@
         $(LibrariesNativeArtifactsPath)package.json;
         $(LibrariesNativeArtifactsPath)dotnet.wasm;
         $(LibrariesNativeArtifactsPath)dotnet.js.symbols;
-        $(LibrariesNativeArtifactsPath)dotnet.timezones.blat;
         $(LibrariesNativeArtifactsPath)*.dat;"
         IsNative="true" />
       <!-- for threaded wasm -->
@@ -225,7 +224,6 @@
       <LibrariesRuntimeFiles
                              Include="
         $(LibrariesNativeArtifactsPath)dotnet.wasm;
-        $(LibrariesNativeArtifactsPath)dotnet.timezones.blat;
         $(LibrariesNativeArtifactsPath)*.dat;"
         IsNative="true" />
 

diff --git a/eng/native/configurecompiler.cmake b/eng/native/configurecompiler.cmake
@@ -586,6 +586,9 @@ if(CLR_CMAKE_TARGET_UNIX)
     add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_ANDROID>)
   elseif(CLR_CMAKE_TARGET_LINUX)
     add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_LINUX>)
+    if(CLR_CMAKE_TARGET_BROWSER)
+      add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_BROWSER>)
+    endif()
     if(CLR_CMAKE_TARGET_LINUX_MUSL)
         add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_LINUX_MUSL>)
     endif()
@@ -599,8 +602,6 @@ if(CLR_CMAKE_TARGET_UNIX)
   endif()
 elseif(CLR_CMAKE_TARGET_WASI)
   add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_WASI>)
-elseif(CLR_CMAKE_TARGET_BROWSER)
-  add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_BROWSER>)
 else(CLR_CMAKE_TARGET_UNIX)
   add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_WINDOWS>)
 endif(CLR_CMAKE_TARGET_UNIX)

diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props
@@ -221,13 +221,13 @@
     <PlatformManifestFileEntry Include="libmono-profiler-browser.a" IsNative="true" />
     <PlatformManifestFileEntry Include="libmono-wasm-eh-js.a" IsNative="true" />
     <PlatformManifestFileEntry Include="libmono-wasm-eh-wasm.a" IsNative="true" />
+    <PlatformManifestFileEntry Include="wasm-bundled-timezones.a" IsNative="true" />
     <PlatformManifestFileEntry Include="dotnet.js" IsNative="true" />
     <PlatformManifestFileEntry Include="dotnet.worker.js" IsNative="true" />
     <PlatformManifestFileEntry Include="dotnet.js.symbols" IsNative="true" />
     <PlatformManifestFileEntry Include="dotnet.d.ts" IsNative="true" />
     <PlatformManifestFileEntry Include="dotnet-legacy.d.ts" IsNative="true" />
     <PlatformManifestFileEntry Include="dotnet.wasm" IsNative="true" />
-    <PlatformManifestFileEntry Include="dotnet.timezones.blat" IsNative="true" />
     <PlatformManifestFileEntry Include="icudt.dat" IsNative="true" />
     <PlatformManifestFileEntry Include="icudt_no_CJK.dat" IsNative="true" />
     <PlatformManifestFileEntry Include="icudt_CJK.dat" IsNative="true" />

diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.GetTimeZoneData.Wasm.cs
@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.InteropServices;
+
+internal static partial class Interop
+{
+    internal static partial class Sys
+    {
+        [LibraryImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_GetTimeZoneData", StringMarshalling = StringMarshalling.Utf8, SetLastError = true)]
+        internal static partial IntPtr GetTimeZoneData(string fileName, out int length);
+    }
+}
diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems
@@ -2143,6 +2143,9 @@
     <Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.GetDefaultTimeZone.AnyMobile.cs" Condition="'$(TargetsAndroid)' == 'true' or '$(TargetsLinuxBionic)' == 'true' or '$(IsiOSLike)' == 'true'">
       <Link>Common\Interop\Unix\System.Native\Interop.GetDefaultTimeZone.AnyMobile.cs</Link>
     </Compile>
+    <Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.GetTimeZoneData.Wasm.cs" Condition="'$(TargetsWasi)' == 'true' or '$(TargetsBrowser)' == 'true'">
+      <Link>Common\src\Interop\Unix\System.Native\Interop.GetTimeZoneData.Wasm.cs</Link>
+    </Compile>
     <Compile Include="$(CommonPath)\Interop\Unix\System.Native\Interop.GetEnv.cs">
       <Link>Common\Interop\Unix\System.Native\Interop.GetEnv.cs</Link>
     </Compile>

diff --git a/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs b/src/libraries/System.Private.CoreLib/src/System/TimeZoneInfo.Unix.NonAndroid.cs
@@ -18,6 +18,11 @@ public sealed partial class TimeZoneInfo
         private const string TimeZoneDirectoryEnvironmentVariable = "TZDIR";
         private const string TimeZoneEnvironmentVariable = "TZ";
 
+#if TARGET_WASI || TARGET_BROWSER
+        // if TZDIR is set, then the embedded TZ data will be ignored and normal unix behavior will be used
+        private static readonly bool UseEmbeddedTzDatabase = string.IsNullOrEmpty(Environment.GetEnvironmentVariable(TimeZoneDirectoryEnvironmentVariable));
+#endif
+
         private static TimeZoneInfo GetLocalTimeZoneCore()
         {
             // Without Registry support, create the TimeZoneInfo from a TZ file
@@ -29,9 +34,31 @@ private static TimeZoneInfoResult TryGetTimeZoneFromLocalMachineCore(string id,
             value = null;
             e = null;
 
+            byte[]? rawData=null;
             string timeZoneDirectory = GetTimeZoneDirectory();
             string timeZoneFilePath = Path.Combine(timeZoneDirectory, id);
-            byte[] rawData;
+
+#if TARGET_WASI || TARGET_BROWSER
+            if (UseEmbeddedTzDatabase)
+            {
+                if(!TryLoadEmbeddedTzFile(timeZoneFilePath, out rawData))
+                {
+                    e = new FileNotFoundException(id, "Embedded TZ data not found");
+                    return TimeZoneInfoResult.TimeZoneNotFoundException;
+                }
+
+                value = GetTimeZoneFromTzData(rawData, id);
+
+                if (value == null)
+                {
+                    e = new InvalidTimeZoneException(SR.Format(SR.InvalidTimeZone_InvalidFileData, id, id));
+                    return TimeZoneInfoResult.InvalidTimeZoneException;
+                }
+
+                return TimeZoneInfoResult.Success;
+            }
+#endif
+
             try
             {
                 rawData = File.ReadAllBytes(timeZoneFilePath);
@@ -74,52 +101,68 @@ private static TimeZoneInfoResult TryGetTimeZoneFromLocalMachineCore(string id,
         /// <remarks>
         /// Lines that start with # are comments and are skipped.
         /// </remarks>
-        private static List<string> GetTimeZoneIds()
+        private static IEnumerable<string> GetTimeZoneIds()
+        {
+            try
+            {
+                var fileName = Path.Combine(GetTimeZoneDirectory(), TimeZoneFileName);
+#if TARGET_WASI || TARGET_BROWSER
+                if (UseEmbeddedTzDatabase)
+                {
+                    if(!TryLoadEmbeddedTzFile(fileName, out var rawData))
+                    {
+                        return Array.Empty<string>();
+                    }
+                    using var blobReader = new StreamReader(new MemoryStream(rawData), Encoding.UTF8);
+                    return ParseTimeZoneIds(blobReader);
+                }
+#endif
+                using var reader = new StreamReader(fileName, Encoding.UTF8);
+                return ParseTimeZoneIds(reader);
+            }
+            catch (IOException) { }
+            catch (UnauthorizedAccessException) { }
+            return Array.Empty<string>();
+        }
+
+        private static List<string> ParseTimeZoneIds(StreamReader reader)
         {
             List<string> timeZoneIds = new List<string>();
 
-            try
+            string? zoneTabFileLine;
+            while ((zoneTabFileLine = reader.ReadLine()) != null)
             {
-                using (StreamReader sr = new StreamReader(Path.Combine(GetTimeZoneDirectory(), TimeZoneFileName), Encoding.UTF8))
+                if (!string.IsNullOrEmpty(zoneTabFileLine) && zoneTabFileLine[0] != '#')
                 {
-                    string? zoneTabFileLine;
-                    while ((zoneTabFileLine = sr.ReadLine()) != null)
+                    // the format of the line is "country-code \t coordinates \t TimeZone Id \t comments"
+
+                    int firstTabIndex = zoneTabFileLine.IndexOf('\t');
+                    if (firstTabIndex >= 0)
                     {
-                        if (!string.IsNullOrEmpty(zoneTabFileLine) && zoneTabFileLine[0] != '#')
+                        int secondTabIndex = zoneTabFileLine.IndexOf('\t', firstTabIndex + 1);
+                        if (secondTabIndex >= 0)
                         {
-                            // the format of the line is "country-code \t coordinates \t TimeZone Id \t comments"
-
-                            int firstTabIndex = zoneTabFileLine.IndexOf('\t');
-                            if (firstTabIndex >= 0)
+                            string timeZoneId;
+                            int startIndex = secondTabIndex + 1;
+                            int thirdTabIndex = zoneTabFileLine.IndexOf('\t', startIndex);
+                            if (thirdTabIndex >= 0)
                             {
-                                int secondTabIndex = zoneTabFileLine.IndexOf('\t', firstTabIndex + 1);
-                                if (secondTabIndex >= 0)
-                                {
-                                    string timeZoneId;
-                                    int startIndex = secondTabIndex + 1;
-                                    int thirdTabIndex = zoneTabFileLine.IndexOf('\t', startIndex);
-                                    if (thirdTabIndex >= 0)
-                                    {
-                                        int length = thirdTabIndex - startIndex;
-                                        timeZoneId = zoneTabFileLine.Substring(startIndex, length);
-                                    }
-                                    else
-                                    {
-                                        timeZoneId = zoneTabFileLine.Substring(startIndex);
-                                    }
+                                int length = thirdTabIndex - startIndex;
+                                timeZoneId = zoneTabFileLine.Substring(startIndex, length);
+                            }
+                            else
+                            {
+                                timeZoneId = zoneTabFileLine.Substring(startIndex);
+                            }
 
-                                    if (!string.IsNullOrEmpty(timeZoneId))
-                                    {
-                                        timeZoneIds.Add(timeZoneId);
-                                    }
-                                }
+                            if (!string.IsNullOrEmpty(timeZoneId))
+                            {
+                                timeZoneIds.Add(timeZoneId);
                             }
                         }
                     }
                 }
             }
-            catch (IOException) { }
-            catch (UnauthorizedAccessException) { }
 
             return timeZoneIds;
         }
@@ -379,6 +422,22 @@ private static bool TryLoadTzFile(string tzFilePath, [NotNullWhen(true)] ref byt
             return false;
         }
 
+#if TARGET_WASI || TARGET_BROWSER
+        private static bool TryLoadEmbeddedTzFile(string name, [NotNullWhen(true)] out byte[]? rawData)
+        {
+            IntPtr bytes = Interop.Sys.GetTimeZoneData(name, out int length);
+            if(bytes == IntPtr.Zero)
+            {
+                rawData = null;
+                return false;
+            }
+
+            rawData = new byte[length];
+            Marshal.Copy(bytes, rawData, 0, length);
+            return true;
+        }
+#endif
+
         /// <summary>
         /// Gets the tzfile raw data for the current 'local' time zone using the following rules.
         ///
@@ -387,6 +446,10 @@ private static bool TryLoadTzFile(string tzFilePath, [NotNullWhen(true)] ref byt
         /// 2. Get the default TZ from the device
         /// 3. Use UTC if all else fails.
         ///
+        /// On WASI / Browser
+        /// 1. if TZDIR is not set, use TZ variable as id to embedded database.
+        /// 2. fall back to unix behavior if TZDIR is set.
+        ///
         /// On all other platforms
         /// 1. Read the TZ environment variable.  If it is set, use it.
         /// 2. Look for the data in /etc/localtime.
@@ -406,6 +469,11 @@ private static bool TryGetLocalTzFile([NotNullWhen(true)] out byte[]? rawData, [
             {
 #if TARGET_IOS || TARGET_TVOS
                 tzVariable = Interop.Sys.GetDefaultTimeZone();
+#elif TARGET_WASI || TARGET_BROWSER
+                if (UseEmbeddedTzDatabase)
+                {
+                    return false; // use UTC
+                }
 #else
                 return
                     TryLoadTzFile("/etc/localtime", ref rawData, ref id) ||
@@ -432,6 +500,24 @@ private static bool TryGetLocalTzFile([NotNullWhen(true)] out byte[]? rawData, [
             {
                 tzFilePath = tzVariable;
             }
+
+#if TARGET_WASI || TARGET_BROWSER
+            if (UseEmbeddedTzDatabase)
+            {
+                // embedded database only supports relative paths
+                if (tzVariable[0] == '/')
+                {
+                    return false;
+                }
+                if(!TryLoadEmbeddedTzFile(tzFilePath, out rawData))
+                {
+                    return false;
+                }
+                id = tzVariable;
+                return true;
+            }
+#endif
+
             return TryLoadTzFile(tzFilePath, ref rawData, ref id);
         }
 

diff --git a/src/mono/mono/utils/mono-dl-wasm.c b/src/mono/mono/utils/mono-dl-wasm.c
@@ -1,5 +1,6 @@
 #include <config.h>
 #include <mono/utils/mono-compiler.h>
+#include <mono/eglib/glib.h>
 
 #if defined (HOST_WASM)
 
@@ -93,3 +94,45 @@ mono_dl_close_handle (MonoDl *module, MonoError *error)
 MONO_EMPTY_SOURCE_FILE (mono_dl_wasm);
 
 #endif
+
+#if defined (HOST_WASM)
+
+static GHashTable *name_to_blob = NULL;
+
+typedef struct {
+	const unsigned char *data;
+	unsigned int size;
+} FileBlob;
+
+int
+mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size)
+{
+	// printf("mono_wasm_add_bundled_file: %s %p %d\n", name, data, size);
+	if(name_to_blob == NULL)
+	{
+		name_to_blob = g_hash_table_new (g_str_hash, g_str_equal);
+	}
+	FileBlob *blob = g_new0 (FileBlob, 1);
+	blob->data = data;
+	blob->size = size;
+	g_hash_table_insert (name_to_blob, (gpointer) name, blob);
+	return 0;
+}
+
+const unsigned char*
+mono_wasm_get_bundled_file (const char *name, int* out_length)
+{
+	FileBlob *blob = (FileBlob *)g_hash_table_lookup (name_to_blob, name);
+	if (blob != NULL)
+	{
+		// printf("mono_wasm_get_bundled_file: %s %p %d \n", name, blob->data, blob->size);
+		*out_length = blob->size;
+		return blob->data;
+	}
+
+	// printf("mono_wasm_get_bundled_file: %s not found \n", name);
+	*out_length = 0;
+	return NULL;
+}
+
+#endif /* HOST_WASM */
diff --git a/src/mono/mono/utils/mono-dl.h b/src/mono/mono/utils/mono-dl.h
@@ -58,5 +58,10 @@ int mono_dl_convert_flags (int mono_flags, int native_flags);
 char* mono_dl_current_error_string (void);
 const char* mono_dl_get_system_dir (void);
 
+#if defined (HOST_WASM)
+int mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size);
+const unsigned char* mono_wasm_get_bundled_file (const char *name, int* out_length);
+#endif /* HOST_WASM */
+
 #endif /* __MONO_UTILS_DL_H__ */
 
diff --git a/src/mono/wasi/Wasi.Build.Tests/BuildTestBase.cs b/src/mono/wasi/Wasi.Build.Tests/BuildTestBase.cs
@@ -360,7 +360,6 @@ static void AssertRuntimePackPath(string buildOutput, string targetFramework)
             {
                 "index.html",
                 mainJS,
-                "dotnet.timezones.blat",
                 "dotnet.wasm",
                 "mono-config.json",
                 "dotnet.js"