diff --git a/eng/Versions.props b/eng/Versions.props index 2295b6e440d51..a194baa33def9 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -140,7 +140,7 @@ These are used as reference assemblies only, so they must not take a ProdCon/source-build version. Insert "RefOnly" to avoid assignment via PVP. --> - 16.8.0 + 16.9.0 $(RefOnlyMicrosoftBuildVersion) $(RefOnlyMicrosoftBuildVersion) $(RefOnlyMicrosoftBuildVersion) diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index b1ca6c3a1ddba..5cad3c540880b 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -255,6 +255,8 @@ + + @@ -276,6 +278,9 @@ + + + diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index b6897fbd2b413..c034763ca0f25 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -3,6 +3,7 @@ + <_WasmBuildNativeCoreDependsOn> @@ -165,7 +166,7 @@ <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == ''">-Oz $(_EmccOptimizationFlagDefault) - $(_EmccOptimizationFlagDefault) + -O0 @@ -173,10 +174,6 @@ <_EmccCommonFlags Include="$(EmccFlags)" /> <_EmccCommonFlags Include="-s DISABLE_EXCEPTION_CATCHING=0" /> <_EmccCommonFlags Include="-g" Condition="'$(WasmNativeStrip)' == 'false'" /> - <_EmccCommonFlags Include="-DENABLE_AOT=1" Condition="'$(RunAOTCompilation)' == 'true'" /> - <_EmccCommonFlags Include="-DDRIVER_GEN=1" Condition="'$(RunAOTCompilation)' == 'true'" /> - <_EmccCommonFlags Include="-DINVARIANT_GLOBALIZATION=1" Condition="'$(InvariantGlobalization)' == 'true'" /> - <_EmccCommonFlags Include="-DLINK_ICALLS=1" Condition="'$(WasmLinkIcalls)' == 'true'" /> <_EmccCommonFlags Include="-v" Condition="'$(EmccVerbose)' != 'false'" /> @@ -186,11 +183,7 @@ - <_WasmObjectsToBuild Include="$(_WasmRuntimePackSrcDir)\*.c" /> - <_WasmObjectsToBuild OutputPath="$(_WasmIntermediateOutputPath)%(FileName).o" /> - <_DotnetJSSrcFile Include="$(_WasmRuntimePackSrcDir)\*.js" /> - <_WasmPInvokeModules Include="%(_WasmNativeFileForLinking.FileName)" /> @@ -227,54 +220,84 @@ <_EmccCFlags Include="$(EmccCompileOptimizationFlag)" /> <_EmccCFlags Include="@(_EmccCommonFlags)" /> + + <_EmccCFlags Include="-DENABLE_AOT=1" Condition="'$(RunAOTCompilation)' == 'true'" /> + <_EmccCFlags Include="-DDRIVER_GEN=1" Condition="'$(RunAOTCompilation)' == 'true'" /> + <_EmccCFlags Include="-DINVARIANT_GLOBALIZATION=1" Condition="'$(InvariantGlobalization)' == 'true'" /> + <_EmccCFlags Include="-DLINK_ICALLS=1" Condition="'$(WasmLinkIcalls)' == 'true'" /> <_EmccCFlags Include="-DCORE_BINDINGS" /> <_EmccCFlags Include="-DGEN_PINVOKE=1" /> + <_EmccCFlags Include=""-I%(_EmccIncludePaths.Identity)"" /> <_EmccCFlags Include="-g" Condition="'$(WasmNativeDebugSymbols)' == 'true'" /> <_EmccCFlags Include="-s EXPORTED_FUNCTIONS='[@(_ExportedFunctions->'"%(Identity)"', ',')]'" Condition="@(_ExportedFunctions->Count()) > 0" /> <_EmccCFlags Include="$(EmccExtraCFlags)" /> + + <_WasmRuntimePackSrcFile Remove="@(_WasmRuntimePackSrcFile)" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)\*.c" /> + <_WasmRuntimePackSrcFile ObjectFile="$(_WasmIntermediateOutputPath)%(FileName).o" /> + <_WasmSourceFileToCompile Include="@(_WasmRuntimePackSrcFile)" /> + <_EmBuilder Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">embuilder.bat + <_EmBuilder Condition="!$([MSBuild]::IsOSPlatform('WINDOWS'))">embuilder.py <_EmccCompileRsp>$(_WasmIntermediateOutputPath)emcc-compile.rsp + + + - + - <_WasmRuntimePackNativeLibs Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)\*.a" /> - <_WasmObjects Include="@(_WasmRuntimePackNativeLibs)" /> - <_WasmObjects Include="@(_WasmObjectsToBuild->'%(OutputPath)')" /> - <_EmccLDFlags Include="$(EmccLinkOptimizationFlag)" /> <_EmccLDFlags Include="@(_EmccCommonFlags)" /> + <_EmccLDFlags Include="-s TOTAL_MEMORY=536870912" /> <_EmccLDFlags Include="$(EmccExtraLDFlags)" /> + + + + + + + <_WasmNativeFileForLinking Include="%(_BitcodeFile.ObjectFile)" /> + <_WasmNativeFileForLinking Include="%(_WasmSourceFileToCompile.ObjectFile)" /> - <_EmccLDFlags Include="--js-library "%(_DotnetJSSrcFile.Identity)"" /> - <_EmccLDFlags Include="--js-library "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library'" /> + + <_WasmNativeFileForLinking Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)\*.a" /> - <_EmccLDFlags Include="--pre-js "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'pre-js'" /> - <_EmccLDFlags Include="--post-js "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'post-js'" /> + <_EmccLinkStepArgs Include="@(_EmccLDFlags)" /> + <_EmccLinkStepArgs Include="--js-library "%(_DotnetJSSrcFile.Identity)"" /> + <_EmccLinkStepArgs Include="--js-library "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library'" /> - <_EmccLDFlags Include=""%(_WasmNativeFileForLinking.Identity)"" /> - <_EmccLDFlags Include=""%(_WasmObjects.Identity)"" /> - <_EmccLDFlags Include="-o "$(_WasmIntermediateOutputPath)dotnet.js"" /> + <_EmccLinkStepArgs Include="--pre-js "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'pre-js'" /> + <_EmccLinkStepArgs Include="--post-js "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'post-js'" /> + + <_EmccLinkStepArgs Include=""%(_WasmNativeFileForLinking.Identity)"" /> + <_EmccLinkStepArgs Include="-o "$(_WasmIntermediateOutputPath)dotnet.js"" /> <_EmccLinkRsp>$(_WasmIntermediateOutputPath)emcc-link.rsp - - + + + + @@ -289,7 +312,7 @@ - $(EmccFlags) -DDRIVER_GEN=1 + $(EmccExtraCFlags) -DDRIVER_GEN=1 void mono_profiler_init_aot (const char *desc)%3B EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_profiler_init_aot (desc)%3B } @@ -405,7 +428,7 @@ EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_ Profilers="$(WasmProfilers)" AotModulesTablePath="$(_WasmIntermediateOutputPath)driver-gen.c" UseLLVM="true" - DisableParallelAot="true" + DisableParallelAot="$(DisableParallelAot)" DedupAssembly="$(_WasmDedupAssembly)" LLVMDebug="dwarfdebug" LLVMPath="$(EmscriptenUpstreamBinPath)" > @@ -420,8 +443,7 @@ EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_ <_AOTAssemblies Include="@(_WasmAssembliesInternal)" Condition="'%(_WasmAssembliesInternal._InternalForceInterpret)' != 'true'" /> <_BitcodeFile Include="%(_WasmAssembliesInternal.LlvmBitcodeFile)" /> - - <_WasmNativeFileForLinking Include="@(_BitcodeFile)" /> + <_BitcodeFile ObjectFile="$(_WasmIntermediateOutputPath)%(FileName).o" /> diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 52d37fe900e0f..cb3876cfc9c36 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -417,8 +417,17 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) try { // run the AOT compiler - Utils.RunProcess(CompilerBinaryPath, string.Join(" ", processArgs), envVariables, assemblyDir, silent: false, - outputMessageImportance: MessageImportance.Low, debugMessageImportance: MessageImportance.Low); + (int exitCode, string output) = Utils.TryRunProcess(CompilerBinaryPath, + string.Join(" ", processArgs), + envVariables, + assemblyDir, + silent: false, + debugMessageImportance: MessageImportance.Low); + if (exitCode != 0) + { + Log.LogError($"Precompiling failed for {assembly}: {output}"); + return false; + } } catch (Exception ex) { diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 696ef14d85e80..ea2aba607dd8e 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Runtime.InteropServices; using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -21,6 +22,49 @@ public static string GetEmbeddedResource(string file) return reader.ReadToEnd(); } + public static (int exitCode, string output) RunShellCommand(string command, + IDictionary envVars, + string workingDir, + MessageImportance debugMessageImportance=MessageImportance.Low) + { + string scriptFileName = CreateTemporaryBatchFile(command); + (string shell, string args) = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? ("cmd", $"/c \"{scriptFileName}\"") + : ("/bin/sh", $"\"{scriptFileName}\""); + + Logger?.LogMessage(debugMessageImportance, $"Running {command} via script {scriptFileName}:"); + Logger?.LogMessage(debugMessageImportance, File.ReadAllText(scriptFileName)); + + return TryRunProcess(shell, + args, + envVars, + workingDir, + silent: false, + debugMessageImportance); + + static string CreateTemporaryBatchFile(string command) + { + string extn = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".cmd" : ".sh"; + string file = Path.Combine(Path.GetTempPath(), $"tmp{Guid.NewGuid():N}{extn}"); + + using StreamWriter sw = new(file); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + sw.WriteLine("setlocal"); + sw.WriteLine("set errorlevel=dummy"); + sw.WriteLine("set errorlevel="); + } + else + { + // Use sh rather than bash, as not all 'nix systems necessarily have Bash installed + sw.WriteLine("#!/bin/sh"); + } + + sw.WriteLine(command); + return file; + } + } + public static string RunProcess( string path, string args = "", @@ -28,10 +72,31 @@ public static string GetEmbeddedResource(string file) string? workingDir = null, bool ignoreErrors = false, bool silent = true, - MessageImportance outputMessageImportance=MessageImportance.High, MessageImportance debugMessageImportance=MessageImportance.High) { - LogInfo($"Running: {path} {args}", debugMessageImportance); + (int exitCode, string output) = TryRunProcess( + path, + args, + envVars, + workingDir, + silent, + debugMessageImportance); + + if (exitCode != 0 && !ignoreErrors) + throw new Exception("Error: Process returned non-zero exit code: " + output); + + return output; + } + + public static (int, string) TryRunProcess( + string path, + string args = "", + IDictionary? envVars = null, + string? workingDir = null, + bool silent = true, + MessageImportance debugMessageImportance=MessageImportance.High) + { + Logger?.LogMessage(debugMessageImportance, $"Running: {path} {args}"); var outputBuilder = new StringBuilder(); var processStartInfo = new ProcessStartInfo { @@ -46,7 +111,7 @@ public static string GetEmbeddedResource(string file) if (workingDir != null) processStartInfo.WorkingDirectory = workingDir; - LogInfo($"Using working directory: {workingDir ?? Environment.CurrentDirectory}", debugMessageImportance); + Logger?.LogMessage(debugMessageImportance, $"Using working directory: {workingDir ?? Environment.CurrentDirectory}"); if (envVars != null) { @@ -68,10 +133,11 @@ public static string GetEmbeddedResource(string file) { lock (s_SyncObj) { + if (string.IsNullOrEmpty(e.Data)) + return; + if (!silent) - { LogWarning(e.Data); - } outputBuilder.AppendLine(e.Data); } }; @@ -79,10 +145,11 @@ public static string GetEmbeddedResource(string file) { lock (s_SyncObj) { + if (string.IsNullOrEmpty(e.Data)) + return; + if (!silent) - { - LogInfo(e.Data, outputMessageImportance); - } + Logger?.LogMessage(debugMessageImportance, e.Data); outputBuilder.AppendLine(e.Data); } }; @@ -90,14 +157,31 @@ public static string GetEmbeddedResource(string file) process.BeginErrorReadLine(); process.WaitForExit(); - if (process.ExitCode != 0) + Logger?.LogMessage(debugMessageImportance, $"Exit code: {process.ExitCode}"); + return (process.ExitCode, outputBuilder.ToString().Trim('\r', '\n')); + } + + internal static string CreateTemporaryBatchFile(string command) + { + string extn = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".cmd" : ".sh"; + string file = Path.Combine(Path.GetTempPath(), $"tmp{Guid.NewGuid():N}{extn}"); + + using StreamWriter sw = new(file); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - Logger?.LogMessage(MessageImportance.High, $"Exit code: {process.ExitCode}"); - if (!ignoreErrors) - throw new Exception("Error: Process returned non-zero exit code: " + outputBuilder); + sw.WriteLine("setlocal"); + sw.WriteLine("set errorlevel=dummy"); + sw.WriteLine("set errorlevel="); } + else + { + // Use sh rather than bash, as not all 'nix systems necessarily have Bash installed + sw.WriteLine("#!/bin/sh"); + } + + sw.WriteLine(command); - return silent ? string.Empty : outputBuilder.ToString().Trim('\r', '\n'); + return file; } #if NETCOREAPP diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs new file mode 100644 index 0000000000000..b175c62c2ee11 --- /dev/null +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -0,0 +1,158 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +#nullable enable + +namespace Microsoft.WebAssembly.Build.Tasks +{ + /// + /// This is meant to *compile* source files only. It is *not* a general purpose + /// `emcc` invocation task. + /// + /// It runs `emcc` for each source file, and with output to `%(SourceFiles.ObjectFile)` + /// + /// + public class EmccCompile : Microsoft.Build.Utilities.Task + { + [NotNull] + [Required] + public ITaskItem[]? SourceFiles { get; set; } + + public ITaskItem[]? EnvironmentVariables { get; set; } + public bool DisableParallelCompile { get; set; } + public string Arguments { get; set; } = string.Empty; + public string? WorkingDirectory { get; set; } + + [Output] + public ITaskItem[]? OutputFiles { get; private set; } + + private string? _tempPath; + + public override bool Execute() + { + if (SourceFiles.Length == 0) + { + Log.LogError($"No SourceFiles to compile"); + return false; + } + + ITaskItem? badItem = SourceFiles.FirstOrDefault(sf => string.IsNullOrEmpty(sf.GetMetadata("ObjectFile"))); + if (badItem != null) + { + Log.LogError($"Source file {badItem.ItemSpec} is missing ObjectFile metadata."); + return false; + } + + IDictionary envVarsDict = GetEnvironmentVariablesDict(); + ConcurrentBag outputItems = new(); + try + { + Log.LogMessage(MessageImportance.Low, "Using environment variables:"); + foreach (var kvp in envVarsDict) + Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}"); + + string workingDir = Environment.CurrentDirectory; + Log.LogMessage(MessageImportance.Low, $"Using working directory: {workingDir}"); + + _tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(_tempPath); + + int allowedParallelism = Math.Min(SourceFiles.Length, Environment.ProcessorCount); +#if false // Enable this when we bump msbuild to 16.1.0 + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); +#endif + + if (DisableParallelCompile || allowedParallelism == 1) + { + foreach (ITaskItem srcItem in SourceFiles) + { + if (!ProcessSourceFile(srcItem)) + return false; + } + } + else + { + ParallelLoopResult result = Parallel.ForEach(SourceFiles, + new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, + (srcItem, state) => + { + if (!ProcessSourceFile(srcItem)) + state.Stop(); + }); + + if (!result.IsCompleted && !Log.HasLoggedErrors) + Log.LogError("Unknown failed occured while compiling"); + } + } + finally + { + if (!string.IsNullOrEmpty(_tempPath)) + Directory.Delete(_tempPath, true); + } + + OutputFiles = outputItems.ToArray(); + return !Log.HasLoggedErrors; + + bool ProcessSourceFile(ITaskItem srcItem) + { + string srcFile = srcItem.ItemSpec; + string objFile = srcItem.GetMetadata("ObjectFile"); + + try + { + string command = $"emcc {Arguments} -c -o {objFile} {srcFile}"; + (int exitCode, string output) = Utils.RunShellCommand(command, envVarsDict, workingDir: Environment.CurrentDirectory); + + if (exitCode != 0) + { + Log.LogError($"Failed to compile {srcFile} -> {objFile}: {output}"); + return false; + } + + ITaskItem newItem = new TaskItem(objFile); + newItem.SetMetadata("SourceFile", srcFile); + outputItems.Add(newItem); + + return true; + } + catch (Exception ex) + { + Log.LogError($"Failed to compile {srcFile} -> {objFile}: {ex.Message}"); + return false; + } + } + } + + private IDictionary GetEnvironmentVariablesDict() + { + Dictionary envVarsDict = new(); + if (EnvironmentVariables == null) + return envVarsDict; + + foreach (var item in EnvironmentVariables) + { + var parts = item.ItemSpec.Split(new char[] {'='}, 2, StringSplitOptions.None); + if (parts.Length == 0) + continue; + + string key = parts[0]; + string value = parts.Length > 1 ? parts[1] : string.Empty; + + envVarsDict[key] = value; + } + + return envVarsDict; + } + } +} diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 660272e268aa4..46c0174148be2 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -3,16 +3,11 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; -using System.Text; -using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; -using System.Reflection; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj index 23074621b8973..b27176e277400 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.csproj @@ -14,6 +14,8 @@ + +