From 0f58db283cd71212da3138af7b46bec5c1e88d64 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Mon, 27 Feb 2023 16:03:10 -0800 Subject: [PATCH 01/28] Add logic to change the encoding and detect .NET SDK and VS language settings. --- src/MSBuild/XMake.cs | 86 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index ade043b1c9a..cc9ef4eace1 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -37,6 +37,8 @@ using Microsoft.Build.Shared.Debugging; using Microsoft.Build.Experimental; using Microsoft.Build.Framework.Telemetry; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; #nullable disable @@ -715,9 +717,9 @@ string[] commandLine inputResultsCaches, outputResultsCache, commandLine)) - { - exitType = ExitType.BuildError; - } + { + exitType = ExitType.BuildError; + } } // end of build DateTime t2 = DateTime.Now; @@ -1244,7 +1246,7 @@ string[] commandLine { if (graphBuildOptions != null) { - graphBuildRequest = new GraphBuildRequestData(new[]{ new ProjectGraphEntryPoint(projectFile, globalProperties) }, targets, null, BuildRequestDataFlags.None, graphBuildOptions); + graphBuildRequest = new GraphBuildRequestData(new[] { new ProjectGraphEntryPoint(projectFile, globalProperties) }, targets, null, BuildRequestDataFlags.None, graphBuildOptions); } else { @@ -1512,7 +1514,7 @@ internal static void SetConsoleUI() Thread thisThread = Thread.CurrentThread; // Eliminate the complex script cultures from the language selection. - thisThread.CurrentUICulture = CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); + thisThread.CurrentUICulture = GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); // Determine if the language can be displayed in the current console codepage, otherwise set to US English int codepage; @@ -1547,6 +1549,74 @@ internal static void SetConsoleUI() #endif } + /// + /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect those variables. + /// To use the correspoding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases. + /// This code introduces a breaking change due to the encoding of the console being changed. + /// If the environment variables are undefined, this function should be a no-op. + /// + /// The custom language that was set by the user for an 'external' tool besides MSBuild. + /// DOTNET_CLI_UI_LANGUAGE > VSLANG. Returns null if none are set. + private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding() + { + CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage(); + if (externalLanguageSetting != null) + { + if ( + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && // Encoding is only an issue on Windows + !externalLanguageSetting.TwoLetterISOLanguageName.Equals("en", StringComparison.InvariantCultureIgnoreCase) && + Environment.OSVersion.Version.Major >= 10 // UTF-8 is only officially supported on 10+. + ) + { + // Setting both encodings causes a change in the CHCP, making it so we dont need to P-Invoke ourselves. + Console.OutputEncoding = Encoding.UTF8; + // If the InputEncoding is not set, the encoding will work in CMD but not in Powershell, as the raw CHCP page won't be changed. + Console.InputEncoding = Encoding.UTF8; + return externalLanguageSetting; + } + else if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return externalLanguageSetting; + } + } + return null; + } + + /// + /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE and VSLANG). + /// Does NOT check System Locale or OS Display Language. + /// Ported from the .NET SDK: https://github.com/dotnet/sdk/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs. + /// + /// The custom language that was set by the user for an 'external' tool besides MSBuild. + /// DOTNET_CLI_UI_LANGUAGE > VSLANG. Returns null if none are set. + private static CultureInfo GetExternalOverriddenUILanguage() + { + // DOTNET_CLI_UI_LANGUAGE= is the main way for users to customize the CLI's UI language via the .NET SDK. + string dotnetCliLanguage = Environment.GetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE"); + if (dotnetCliLanguage != null) + { + try + { + return new CultureInfo(dotnetCliLanguage); + } + catch (CultureNotFoundException) { } + } + + // VSLANG= is set by Visual Studio. + string vsLang = Environment.GetEnvironmentVariable("VSLANG"); + if (vsLang != null && int.TryParse(vsLang, out int vsLcid)) + { + try + { + return new CultureInfo(vsLcid); + } + catch (ArgumentOutOfRangeException) { } + catch (CultureNotFoundException) { } + } + + return null; + } + /// /// Gets all specified switches, from the command line, as well as all /// response files, including the auto-response file. @@ -2406,7 +2476,7 @@ internal static GraphBuildOptions ProcessGraphBuildSwitch(string[] parameters) if (parameter.Trim().Equals("NoBuild", StringComparison.OrdinalIgnoreCase)) { - options = options with {Build = false}; + options = options with { Build = false }; } else { @@ -2893,7 +2963,7 @@ DirectoryGetFiles getFiles } // if there are no project, solution filter, or solution files in the directory, we can't build else if (actualProjectFiles.Count == 0 && - actualSolutionFiles.Count== 0 && + actualSolutionFiles.Count == 0 && solutionFilterFiles.Count == 0) { InitializationException.Throw("MissingProjectError", null, null, false); @@ -3183,7 +3253,7 @@ private static void ProcessBinaryLogger(string[] binaryLoggerParameters, List Date: Mon, 27 Feb 2023 16:32:06 -0800 Subject: [PATCH 02/28] Restore the original encoding so msbuild does not impact encoding of other programs on the same console. Note that we could probably do this better. I tried to create a function to do so but because the SetConsoleUI is static this was the least ugly/simple way to do it. Open to other proposals. --- src/MSBuild/XMake.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index cc9ef4eace1..204da17a29a 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -570,6 +570,9 @@ string[] commandLine VerifyThrowSupportedOS(); // Setup the console UI. + Encoding originalOutputEncoding = Console.OutputEncoding; + Encoding originalInputEncoding = Console.InputEncoding; + CultureInfo originalThreadCulture = Thread.CurrentThread.CurrentUICulture; SetConsoleUI(); // reset the application state for this new build @@ -738,6 +741,10 @@ string[] commandLine // if there was no need to start the build e.g. because /help was triggered // do nothing } + + // The encoding may be changed to support non-en characters for environment variables set by external tools. We don't want to impact other programs on the console. + Console.OutputEncoding = originalOutputEncoding; + Console.InputEncoding = originalInputEncoding; } /********************************************************************************************************************** * WARNING: Do NOT add any more catch blocks below! Exceptions should be caught as close to their point of origin as From e4595a97a47fbdf457a7a01bfb715172fd769247 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Mon, 27 Feb 2023 16:43:52 -0800 Subject: [PATCH 03/28] Add a unit test for external ui language overrides --- src/MSBuild.UnitTests/XMake_Tests.cs | 33 +++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 5849fb8f3c8..58850021cb5 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -20,6 +20,8 @@ using System.Reflection; using Microsoft.Build.Utilities; using Microsoft.Build.Logging; +using Newtonsoft.Json.Linq; +using System.Text; #nullable disable @@ -518,7 +520,7 @@ public void Help(string indicator) #if FEATURE_GET_COMMANDLINE @$"c:\bin\msbuild.exe {indicator} " #else - new [] {@"c:\bin\msbuild.exe", indicator} + new[] { @"c:\bin\msbuild.exe", indicator } #endif ).ShouldBe(MSBuildApp.ExitType.Success); } @@ -648,6 +650,35 @@ public void SetConsoleUICulture() thisThread.CurrentUICulture = originalUICulture; } + + [Fact] + public void ConsoleUIRespectsSDKLanguage() + { + const string DOTNET_CLI_UI_LANGUAGE = nameof(DOTNET_CLI_UI_LANGUAGE); + + // Save the current environment info so it can be restored. + var originalUILanguage = Environment.GetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE); + Encoding originalOutputEncoding = Console.OutputEncoding; + Encoding originalInputEncoding = Console.InputEncoding; + Thread thisThread = Thread.CurrentThread; + CultureInfo originalUICulture = thisThread.CurrentUICulture; + + // Set the UI language based on the SDK environment var. + Environment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, "ja"); // japanese chose arbitrarily. + + MSBuildApp.SetConsoleUI(); + + Assert.Equal(thisThread.CurrentUICulture, new CultureInfo("ja")); + Assert.Equal(65001, System.Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. + + // Restore the current UI culture back to the way it was at the beginning of this unit test. + thisThread.CurrentUICulture = originalUICulture; + Environment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, originalUILanguage); + // MSbuild should also restore the encoding upon exit, but we don't create that context here. + Console.OutputEncoding = originalOutputEncoding; + Console.InputEncoding = originalInputEncoding; + } + #if FEATURE_SYSTEM_CONFIGURATION /// /// Invalid configuration file should not dump stack. From fa715d1d22600f5d57b2538c2ba69e4a4623c5e6 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Mon, 27 Feb 2023 16:48:43 -0800 Subject: [PATCH 04/28] Fix the merge conflict that vscode did not fix (grumpiness) --- src/MSBuild.UnitTests/XMake_Tests.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 8ca735498c5..9441601b99a 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -517,11 +517,7 @@ public void Help(string indicator) #if FEATURE_GET_COMMANDLINE @$"c:\bin\msbuild.exe {indicator} ") #else -<<<<<<< HEAD - new[] { @"c:\bin\msbuild.exe", indicator } -======= new[] { @"c:\bin\msbuild.exe", indicator }) ->>>>>>> upstream/main #endif .ShouldBe(MSBuildApp.ExitType.Success); } From cc30ce9c0267607c7be1503882400f71271c8c92 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Tue, 28 Feb 2023 09:18:02 -0800 Subject: [PATCH 05/28] Use langword=null for Sandcastle and VsDocMan Co-authored-by: Rainer Sigwald --- src/MSBuild/XMake.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 05551dd4ef1..6e5ba4f959e 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1684,7 +1684,7 @@ internal static void SetConsoleUI() /// If the environment variables are undefined, this function should be a no-op. /// /// The custom language that was set by the user for an 'external' tool besides MSBuild. - /// DOTNET_CLI_UI_LANGUAGE > VSLANG. Returns null if none are set. + /// DOTNET_CLI_UI_LANGUAGE > VSLANG. Returns if none are set. private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding() { CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage(); From 1058b5970c55c5023056c790531b7fd7fc7796d1 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Mon, 6 Mar 2023 13:43:56 -0800 Subject: [PATCH 06/28] Respond to feedback. But it's not done yet. --- src/MSBuild.UnitTests/XMake_Tests.cs | 31 +++++++++++++++++----------- src/MSBuild/XMake.cs | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 9441601b99a..4105f40bb96 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -9,6 +9,7 @@ using System.IO.Compression; using System.Linq; using System.Reflection; +using System.Text; using System.Threading; using Microsoft.Build.CommandLine; using Microsoft.Build.Framework; @@ -647,28 +648,34 @@ public void SetConsoleUICulture() public void ConsoleUIRespectsSDKLanguage() { const string DOTNET_CLI_UI_LANGUAGE = nameof(DOTNET_CLI_UI_LANGUAGE); - + using TestEnvironment testEnvironment = TestEnvironment.Create(); // Save the current environment info so it can be restored. var originalUILanguage = Environment.GetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE); + Encoding originalOutputEncoding = Console.OutputEncoding; Encoding originalInputEncoding = Console.InputEncoding; Thread thisThread = Thread.CurrentThread; CultureInfo originalUICulture = thisThread.CurrentUICulture; - // Set the UI language based on the SDK environment var. - Environment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, "ja"); // japanese chose arbitrarily. + try + { + // Set the UI language based on the SDK environment var. + testEnvironment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, "ja"); // japanese chose arbitrarily. - MSBuildApp.SetConsoleUI(); + MSBuildApp.SetConsoleUI(); - Assert.Equal(thisThread.CurrentUICulture, new CultureInfo("ja")); - Assert.Equal(65001, System.Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. + Assert.Equal(thisThread.CurrentUICulture, new CultureInfo("ja")); + Assert.Equal(65001, Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. + } + finally + { + // Restore the current UI culture back to the way it was at the beginning of this unit test. + thisThread.CurrentUICulture = originalUICulture; - // Restore the current UI culture back to the way it was at the beginning of this unit test. - thisThread.CurrentUICulture = originalUICulture; - Environment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, originalUILanguage); - // MSbuild should also restore the encoding upon exit, but we don't create that context here. - Console.OutputEncoding = originalOutputEncoding; - Console.InputEncoding = originalInputEncoding; + // MSbuild should also restore the encoding upon exit, but we don't create that context here. + Console.OutputEncoding = originalOutputEncoding; + Console.InputEncoding = originalInputEncoding; + } } #if FEATURE_SYSTEM_CONFIGURATION diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 6e5ba4f959e..ec9f2c17386 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1713,7 +1713,7 @@ private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncodi /// /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE and VSLANG). /// Does NOT check System Locale or OS Display Language. - /// Ported from the .NET SDK: https://github.com/dotnet/sdk/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs. + /// Ported from the .NET SDK: https://github.com/dotnet/sdk/blob/4846f59fe168a343acfb84841f323fd47dd0e72c/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs#L53. /// /// The custom language that was set by the user for an 'external' tool besides MSBuild. /// DOTNET_CLI_UI_LANGUAGE > VSLANG. Returns null if none are set. From 01b381b69cfee78453d2ec64122e2587d37ccdfb Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 8 Mar 2023 15:38:16 -0800 Subject: [PATCH 07/28] Limit the scope of the breaking change so windows who dont support utf8 dont get it by default unless opted in and introduce encoding restorer to prevent msbuild from changing the encoding. --- .../Utilities/AutomaticEncodingRestorer.cs | 75 +++++++++++++++++++ src/MSBuild.UnitTests/XMake_Tests.cs | 6 +- src/MSBuild/XMake.cs | 40 ++++++++-- 3 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 src/Build/Utilities/AutomaticEncodingRestorer.cs diff --git a/src/Build/Utilities/AutomaticEncodingRestorer.cs b/src/Build/Utilities/AutomaticEncodingRestorer.cs new file mode 100644 index 00000000000..ed22352cff3 --- /dev/null +++ b/src/Build/Utilities/AutomaticEncodingRestorer.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Security; +using System.Text; + +namespace Microsoft.Build.Utilities +{ + /// + /// Ported from https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/dotnet/AutomaticEncodingRestorer.cs. + /// A program can change the encoding of the console which would affect other programs. + /// We would prefer to have a pattern where the program does not affect encoding of other programs. + /// Create this class in a function akin to Main and let it manage the console encoding resources to return it to the state before execution upon destruction. + /// + public class AutomaticEncodingRestorer : IDisposable + { + private Encoding? _originalOutputEncoding = null; + private Encoding? _originalInputEncoding = null; + + private bool outputEncodingAccessible = false; + private bool inputEncodingAccessible = false; + + public AutomaticEncodingRestorer() + { + try + { + if ( +#if NET7_0_OR_GREATER + !OperatingSystem.IsIOS() && !OperatingSystem.IsAndroid() && !OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs. +#else + RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // Windows is the only platform where we need to change the encoding as other platforms are UTF 8 by default. +#endif + { + _originalOutputEncoding = Console.OutputEncoding; + outputEncodingAccessible = true; + if ( +#if NET7_0_OR_GREATER + !OperatingSystem.IsBrowser()) // Input Encoding is also unavailable in this platform. +#else + RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) +#endif + { + _originalInputEncoding = Console.InputEncoding; + inputEncodingAccessible = true; + } + } + } + catch (Exception ex) when (ex is IOException || ex is SecurityException) + { + // The encoding is unavailable. Do nothing. + } + } + + public void Dispose() + { + try + { + if (outputEncodingAccessible && _originalOutputEncoding != null) + { + Console.OutputEncoding = _originalOutputEncoding; + } + if (inputEncodingAccessible && _originalInputEncoding != null) + { + Console.InputEncoding = _originalInputEncoding; + } + } + catch (Exception ex) when (ex is IOException || ex is SecurityException) + { + // The encoding is unavailable. Do nothing. + } + } + } +} diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 4105f40bb96..34a6f26386d 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -9,8 +9,8 @@ using System.IO.Compression; using System.Linq; using System.Reflection; -using System.Text; using System.Threading; +using Newtonsoft.Json.Linq; using Microsoft.Build.CommandLine; using Microsoft.Build.Framework; using Microsoft.Build.Logging; @@ -652,8 +652,8 @@ public void ConsoleUIRespectsSDKLanguage() // Save the current environment info so it can be restored. var originalUILanguage = Environment.GetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE); - Encoding originalOutputEncoding = Console.OutputEncoding; - Encoding originalInputEncoding = Console.InputEncoding; + var originalOutputEncoding = Console.OutputEncoding; + var originalInputEncoding = Console.InputEncoding; Thread thisThread = Thread.CurrentThread; CultureInfo originalUICulture = thisThread.CurrentUICulture; diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index ec9f2c17386..609d7058795 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -41,6 +41,8 @@ using Microsoft.Build.Internal; using Microsoft.Build.Logging.LiveLogger; using System.Runtime.InteropServices; +using Microsoft.Win32; +using Microsoft.Build.Utilities; #nullable disable @@ -666,8 +668,7 @@ public static ExitType Execute( VerifyThrowSupportedOS(); // Setup the console UI. - Encoding originalOutputEncoding = Console.OutputEncoding; - Encoding originalInputEncoding = Console.InputEncoding; + using AutomaticEncodingRestorer _ = new(); CultureInfo originalThreadCulture = Thread.CurrentThread.CurrentUICulture; SetConsoleUI(); @@ -838,8 +839,8 @@ public static ExitType Execute( } // The encoding may be changed to support non-en characters for environment variables set by external tools. We don't want to impact other programs on the console. - Console.OutputEncoding = originalOutputEncoding; - Console.InputEncoding = originalInputEncoding; + //Console.OutputEncoding = originalOutputEncoding; + //Console.InputEncoding = originalInputEncoding; } /********************************************************************************************************************** * WARNING: Do NOT add any more catch blocks below! Exceptions should be caught as close to their point of origin as @@ -1691,9 +1692,8 @@ private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncodi if (externalLanguageSetting != null) { if ( - RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && // Encoding is only an issue on Windows !externalLanguageSetting.TwoLetterISOLanguageName.Equals("en", StringComparison.InvariantCultureIgnoreCase) && - Environment.OSVersion.Version.Major >= 10 // UTF-8 is only officially supported on 10+. + CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() ) { // Setting both encodings causes a change in the CHCP, making it so we dont need to P-Invoke ourselves. @@ -1710,10 +1710,36 @@ private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncodi return null; } + private static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version.Major >= 10) // UTF-8 is only officially supported on 10+. + { + try + { + using RegistryKey windowsVersionRegistry = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion"); + var buildNumber = windowsVersionRegistry.GetValue("CurrentBuildNumber").ToString(); + const int buildNumberThatOfficialySupportsUTF8 = 18363; + return int.Parse(buildNumber) >= buildNumberThatOfficialySupportsUTF8 || ForceUniversalEncodingOptInEnabled(); + } + catch (Exception ex) when (ex is SecurityException || ex is ObjectDisposedException) + { + // We don't want to break those in VS on older versions of Windows with a non-en language. + // Allow those without registry permissions to force the encoding, however. + return ForceUniversalEncodingOptInEnabled(); + } + } + return false; + } + + private static bool ForceUniversalEncodingOptInEnabled() + { + return String.Equals(Environment.GetEnvironmentVariable("DOTNET_CLI_FORCE_UTF8_ENCODING"), "true", StringComparison.OrdinalIgnoreCase); + } + /// /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE and VSLANG). /// Does NOT check System Locale or OS Display Language. - /// Ported from the .NET SDK: https://github.com/dotnet/sdk/blob/4846f59fe168a343acfb84841f323fd47dd0e72c/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs#L53. + /// Ported from the .NET SDK: https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs /// /// The custom language that was set by the user for an 'external' tool besides MSBuild. /// DOTNET_CLI_UI_LANGUAGE > VSLANG. Returns null if none are set. From 8487af66b27970a8976e3981528d02bc4f3d228c Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 8 Mar 2023 15:42:20 -0800 Subject: [PATCH 08/28] Remove comment that I didn't hit ctrl S on, automatic encoding restorer takes care of this --- src/MSBuild/XMake.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 609d7058795..13a77520ee4 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -37,7 +37,6 @@ using Microsoft.Build.Shared.Debugging; using Microsoft.Build.Experimental; using Microsoft.Build.Framework.Telemetry; -using System.Runtime.CompilerServices; using Microsoft.Build.Internal; using Microsoft.Build.Logging.LiveLogger; using System.Runtime.InteropServices; @@ -837,10 +836,6 @@ public static ExitType Execute( // if there was no need to start the build e.g. because /help was triggered // do nothing } - - // The encoding may be changed to support non-en characters for environment variables set by external tools. We don't want to impact other programs on the console. - //Console.OutputEncoding = originalOutputEncoding; - //Console.InputEncoding = originalInputEncoding; } /********************************************************************************************************************** * WARNING: Do NOT add any more catch blocks below! Exceptions should be caught as close to their point of origin as From 747ff0b8c6a853ea484640e669d5ec0f139f7615 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 8 Mar 2023 16:45:24 -0800 Subject: [PATCH 09/28] Move encoding restorer to a non public API Co-Authored-By: Forgind <12969783+Forgind@users.noreply.github.com> --- src/Build/Microsoft.Build.csproj | 1 + src/MSBuild/AutomaticEncodingRestorer.cs | 76 ++++++++++++++++++++++++ src/MSBuild/MSBuild.csproj | 18 +++--- src/MSBuild/XMake.cs | 1 - 4 files changed, 87 insertions(+), 9 deletions(-) create mode 100644 src/MSBuild/AutomaticEncodingRestorer.cs diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index c2757e1ff4d..dcb3bfb7ac7 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -162,6 +162,7 @@ + diff --git a/src/MSBuild/AutomaticEncodingRestorer.cs b/src/MSBuild/AutomaticEncodingRestorer.cs new file mode 100644 index 00000000000..4c70147dccf --- /dev/null +++ b/src/MSBuild/AutomaticEncodingRestorer.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Security; +using System.Text; + +namespace Microsoft.Build.CommandLine +{ + /// + /// Ported from https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/dotnet/AutomaticEncodingRestorer.cs. + /// A program can change the encoding of the console which would affect other programs. + /// We would prefer to have a pattern where the program does not affect encoding of other programs. + /// Create this class in a function akin to Main and let it manage the console encoding resources to return it to the state before execution upon destruction. + /// + public class AutomaticEncodingRestorer : IDisposable + { + private Encoding? _originalOutputEncoding = null; + private Encoding? _originalInputEncoding = null; + + private bool outputEncodingAccessible = false; + private bool inputEncodingAccessible = false; + + public AutomaticEncodingRestorer() + { + try + { + if ( +#if NET7_0_OR_GREATER + !OperatingSystem.IsIOS() && !OperatingSystem.IsAndroid() && !OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs. +#else + RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // Windows is the only platform where we need to change the encoding as other platforms are UTF 8 by default. +#endif + { + _originalOutputEncoding = Console.OutputEncoding; + outputEncodingAccessible = true; + if ( +#if NET7_0_OR_GREATER + !OperatingSystem.IsBrowser()) // Input Encoding is also unavailable in this platform. +#else + RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) +#endif + { + _originalInputEncoding = Console.InputEncoding; + inputEncodingAccessible = true; + } + } + } + catch (Exception ex) when (ex is IOException || ex is SecurityException) + { + // The encoding is unavailable. Do nothing. + } + } + + public void Dispose() + { + try + { + if (outputEncodingAccessible && _originalOutputEncoding != null) + { + Console.OutputEncoding = _originalOutputEncoding; + } + if (inputEncodingAccessible && _originalInputEncoding != null) + { + Console.InputEncoding = _originalInputEncoding; + } + } + catch (Exception ex) when (ex is IOException || ex is SecurityException) + { + // The encoding is unavailable. Do nothing. + } + } + } +} diff --git a/src/MSBuild/MSBuild.csproj b/src/MSBuild/MSBuild.csproj index 345d27ecf51..039fe2e452a 100644 --- a/src/MSBuild/MSBuild.csproj +++ b/src/MSBuild/MSBuild.csproj @@ -1,4 +1,4 @@ - + @@ -48,7 +48,8 @@ true false - full + full + $(DefineConstants);MSBUILDENTRYPOINTEXE @@ -162,6 +163,7 @@ true + true @@ -217,13 +219,13 @@ - - - - [2.0.3] - - + + + [2.0.3] + + diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 13a77520ee4..eb4fc31baa8 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -41,7 +41,6 @@ using Microsoft.Build.Logging.LiveLogger; using System.Runtime.InteropServices; using Microsoft.Win32; -using Microsoft.Build.Utilities; #nullable disable From 3b3b23dba53371443fd83b8d7ef9681714fcb1cd Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 8 Mar 2023 16:56:44 -0800 Subject: [PATCH 10/28] Use a name instead of _ for the discard variable bc the compiler gets confused --- src/MSBuild/XMake.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index eb4fc31baa8..c48f0e8a8a4 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -666,7 +666,7 @@ public static ExitType Execute( VerifyThrowSupportedOS(); // Setup the console UI. - using AutomaticEncodingRestorer _ = new(); + using AutomaticEncodingRestorer discardedEncodingRestorer = new(); CultureInfo originalThreadCulture = Thread.CurrentThread.CurrentUICulture; SetConsoleUI(); From d52714bf68280669ba46ecafa4fb000c721b8c8f Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 8 Mar 2023 17:09:27 -0800 Subject: [PATCH 11/28] Dont use runtimeinfo as its not in net 472 full framework even tho its doced as being in net 472 --- src/MSBuild/AutomaticEncodingRestorer.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/MSBuild/AutomaticEncodingRestorer.cs b/src/MSBuild/AutomaticEncodingRestorer.cs index 4c70147dccf..ed4e58df54d 100644 --- a/src/MSBuild/AutomaticEncodingRestorer.cs +++ b/src/MSBuild/AutomaticEncodingRestorer.cs @@ -31,7 +31,7 @@ public AutomaticEncodingRestorer() #if NET7_0_OR_GREATER !OperatingSystem.IsIOS() && !OperatingSystem.IsAndroid() && !OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs. #else - RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // Windows is the only platform where we need to change the encoding as other platforms are UTF 8 by default. + IsWindowsOS()) // Windows is the only platform where we need to change the encoding as other platforms are UTF 8 by default. #endif { _originalOutputEncoding = Console.OutputEncoding; @@ -40,7 +40,7 @@ public AutomaticEncodingRestorer() #if NET7_0_OR_GREATER !OperatingSystem.IsBrowser()) // Input Encoding is also unavailable in this platform. #else - RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + IsWindowsOS()) #endif { _originalInputEncoding = Console.InputEncoding; @@ -72,5 +72,20 @@ public void Dispose() // The encoding is unavailable. Do nothing. } } + + /// + /// RuntimeInformation.IsOSPlatform(OSPlatform.Windows) is supposed to be available in net472 but apparently it isnt part of full framework so we can't use it. + /// + /// + private bool IsWindowsOS() + { + string windir = Environment.GetEnvironmentVariable("windir"); + if (!string.IsNullOrEmpty(windir) && windir.Contains(@"\") && Directory.Exists(windir)) + { + return true; + } + return false; + } + } } From b20919f38a2b7e1cd1a4b35e57229d17cfffe791 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 9 Mar 2023 11:10:51 -0800 Subject: [PATCH 12/28] Move the encoding restorer so it's not part of a public api and builds for net 472 --- src/Build/Microsoft.Build.csproj | 1 - .../Utilities/AutomaticEncodingRestorer.cs | 75 ------------------- src/MSBuild/AutomaticEncodingRestorer.cs | 12 +-- 3 files changed, 4 insertions(+), 84 deletions(-) delete mode 100644 src/Build/Utilities/AutomaticEncodingRestorer.cs diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index dcb3bfb7ac7..c2757e1ff4d 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -162,7 +162,6 @@ - diff --git a/src/Build/Utilities/AutomaticEncodingRestorer.cs b/src/Build/Utilities/AutomaticEncodingRestorer.cs deleted file mode 100644 index ed22352cff3..00000000000 --- a/src/Build/Utilities/AutomaticEncodingRestorer.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Security; -using System.Text; - -namespace Microsoft.Build.Utilities -{ - /// - /// Ported from https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/dotnet/AutomaticEncodingRestorer.cs. - /// A program can change the encoding of the console which would affect other programs. - /// We would prefer to have a pattern where the program does not affect encoding of other programs. - /// Create this class in a function akin to Main and let it manage the console encoding resources to return it to the state before execution upon destruction. - /// - public class AutomaticEncodingRestorer : IDisposable - { - private Encoding? _originalOutputEncoding = null; - private Encoding? _originalInputEncoding = null; - - private bool outputEncodingAccessible = false; - private bool inputEncodingAccessible = false; - - public AutomaticEncodingRestorer() - { - try - { - if ( -#if NET7_0_OR_GREATER - !OperatingSystem.IsIOS() && !OperatingSystem.IsAndroid() && !OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs. -#else - RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // Windows is the only platform where we need to change the encoding as other platforms are UTF 8 by default. -#endif - { - _originalOutputEncoding = Console.OutputEncoding; - outputEncodingAccessible = true; - if ( -#if NET7_0_OR_GREATER - !OperatingSystem.IsBrowser()) // Input Encoding is also unavailable in this platform. -#else - RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) -#endif - { - _originalInputEncoding = Console.InputEncoding; - inputEncodingAccessible = true; - } - } - } - catch (Exception ex) when (ex is IOException || ex is SecurityException) - { - // The encoding is unavailable. Do nothing. - } - } - - public void Dispose() - { - try - { - if (outputEncodingAccessible && _originalOutputEncoding != null) - { - Console.OutputEncoding = _originalOutputEncoding; - } - if (inputEncodingAccessible && _originalInputEncoding != null) - { - Console.InputEncoding = _originalInputEncoding; - } - } - catch (Exception ex) when (ex is IOException || ex is SecurityException) - { - // The encoding is unavailable. Do nothing. - } - } - } -} diff --git a/src/MSBuild/AutomaticEncodingRestorer.cs b/src/MSBuild/AutomaticEncodingRestorer.cs index ed4e58df54d..ba4acfda90b 100644 --- a/src/MSBuild/AutomaticEncodingRestorer.cs +++ b/src/MSBuild/AutomaticEncodingRestorer.cs @@ -31,7 +31,7 @@ public AutomaticEncodingRestorer() #if NET7_0_OR_GREATER !OperatingSystem.IsIOS() && !OperatingSystem.IsAndroid() && !OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs. #else - IsWindowsOS()) // Windows is the only platform where we need to change the encoding as other platforms are UTF 8 by default. + IsWindowsOS()) // Windows is the only platform where we need to change the encoding as other platforms are UTF 8 by default, so for now its the only one required to restore. #endif { _originalOutputEncoding = Console.OutputEncoding; @@ -74,18 +74,14 @@ public void Dispose() } /// + /// Return whether the running OS is windows for net472. /// RuntimeInformation.IsOSPlatform(OSPlatform.Windows) is supposed to be available in net472 but apparently it isnt part of full framework so we can't use it. /// /// private bool IsWindowsOS() { - string windir = Environment.GetEnvironmentVariable("windir"); - if (!string.IsNullOrEmpty(windir) && windir.Contains(@"\") && Directory.Exists(windir)) - { - return true; - } - return false; + string? windir = Environment.GetEnvironmentVariable("windir"); + return !string.IsNullOrEmpty(windir) && windir.Contains(@"\") && Directory.Exists(windir); } - } } From 8d0e14b7ce85364bbdef6605a9370a07f7e09a0b Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 9 Mar 2023 11:47:09 -0800 Subject: [PATCH 13/28] Consider that full framework must set all culture variables for correct behavior --- src/MSBuild/XMake.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index c48f0e8a8a4..6a01bf028e0 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1639,6 +1639,11 @@ internal static void SetConsoleUI() // Eliminate the complex script cultures from the language selection. thisThread.CurrentUICulture = GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); + // For full framework, both the above and below must be set. This is not true in core, but it is a no op in core. + // // https://learn.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.defaultthreadcurrentculture?redirectedfrom=MSDN&view=net-7.0#remarks + CultureInfo.CurrentCulture = thisThread.CurrentUICulture; + CultureInfo.CurrentUICulture = thisThread.CurrentUICulture; + CultureInfo.DefaultThreadCurrentUICulture = thisThread.CurrentUICulture; // Determine if the language can be displayed in the current console codepage, otherwise set to US English int codepage; From d96551c0a53b96bb6f836739dd1dee39c3d51bed Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Mon, 13 Mar 2023 10:03:39 -0700 Subject: [PATCH 14/28] Try to make full framework correctly change the culture :( --- src/MSBuild.UnitTests/XMake_Tests.cs | 2 +- src/MSBuild/XMake.cs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 34a6f26386d..b6c0dffe140 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -664,7 +664,7 @@ public void ConsoleUIRespectsSDKLanguage() MSBuildApp.SetConsoleUI(); - Assert.Equal(thisThread.CurrentUICulture, new CultureInfo("ja")); + Assert.Equal(new CultureInfo("ja"), thisThread.CurrentUICulture); Assert.Equal(65001, Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. } finally diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 6a01bf028e0..4eb29d53fef 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1638,19 +1638,19 @@ internal static void SetConsoleUI() Thread thisThread = Thread.CurrentThread; // Eliminate the complex script cultures from the language selection. - thisThread.CurrentUICulture = GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); + var desiredCulture = GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); + thisThread.CurrentUICulture = desiredCulture; // For full framework, both the above and below must be set. This is not true in core, but it is a no op in core. - // // https://learn.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.defaultthreadcurrentculture?redirectedfrom=MSDN&view=net-7.0#remarks - CultureInfo.CurrentCulture = thisThread.CurrentUICulture; - CultureInfo.CurrentUICulture = thisThread.CurrentUICulture; - CultureInfo.DefaultThreadCurrentUICulture = thisThread.CurrentUICulture; + // https://learn.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.defaultthreadcurrentculture?redirectedfrom=MSDN&view=net-7.0#remarks + CultureInfo.CurrentUICulture = desiredCulture; + CultureInfo.DefaultThreadCurrentUICulture = desiredCulture; // Determine if the language can be displayed in the current console codepage, otherwise set to US English int codepage; try { - codepage = System.Console.OutputEncoding.CodePage; + codepage = Console.OutputEncoding.CodePage; } catch (NotSupportedException) { From 9b1c074ec341b77720f6c13135338772a5b69466 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Mon, 13 Mar 2023 13:47:02 -0700 Subject: [PATCH 15/28] Check to see if the windows version of build machines doesnt support utf 8 --- src/MSBuild.UnitTests/XMake_Tests.cs | 8 ++++++-- src/MSBuild/XMake.cs | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index b6c0dffe140..4c33f31a41c 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -20,6 +20,7 @@ using Shouldly; using Xunit; using Xunit.Abstractions; +using System.Runtime.InteropServices; #nullable disable @@ -664,8 +665,11 @@ public void ConsoleUIRespectsSDKLanguage() MSBuildApp.SetConsoleUI(); - Assert.Equal(new CultureInfo("ja"), thisThread.CurrentUICulture); - Assert.Equal(65001, Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || MSBuildApp.CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()) + { + Assert.Equal(new CultureInfo("ja"), thisThread.CurrentUICulture); + Assert.Equal(65001, Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. + } } finally { diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 4eb29d53fef..f89cd7146f3 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1709,7 +1709,7 @@ private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncodi return null; } - private static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() + public static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version.Major >= 10) // UTF-8 is only officially supported on 10+. { From 5a0629289839e1ed41d3a90fc58f9d8a08020727 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Mon, 13 Mar 2023 16:23:18 -0700 Subject: [PATCH 16/28] Dont look for vslang because vs can manage the lang settings itself already and also fix a test to use en because bild machines are apparently in french sometimes??? --- src/MSBuild.UnitTests/XMake_Tests.cs | 3 +++ src/MSBuild/XMake.cs | 24 +++++++----------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 4c33f31a41c..736c7d920f1 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -838,6 +838,9 @@ public void TestEnvironmentTest() [Fact] public void MSBuildEngineLogger() { + using TestEnvironment testEnvironment = TestEnvironment.Create(); + testEnvironment.SetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE", "en"); // build machines may have other values. + string oldValueForMSBuildLoadMicrosoftTargetsReadOnly = Environment.GetEnvironmentVariable("MSBuildLoadMicrosoftTargetsReadOnly"); string projectString = "" + diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index f89cd7146f3..bf311e39537 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1678,13 +1678,15 @@ internal static void SetConsoleUI() } /// - /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect those variables. + /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect the SDK variable. /// To use the correspoding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases. /// This code introduces a breaking change due to the encoding of the console being changed. /// If the environment variables are undefined, this function should be a no-op. /// - /// The custom language that was set by the user for an 'external' tool besides MSBuild. - /// DOTNET_CLI_UI_LANGUAGE > VSLANG. Returns if none are set. + /// + /// The custom language that was set by the user for an 'external' tool besides MSBuild. + /// Returns if none are set. + /// private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding() { CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage(); @@ -1736,12 +1738,12 @@ private static bool ForceUniversalEncodingOptInEnabled() } /// - /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE and VSLANG). + /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE.) /// Does NOT check System Locale or OS Display Language. /// Ported from the .NET SDK: https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs /// /// The custom language that was set by the user for an 'external' tool besides MSBuild. - /// DOTNET_CLI_UI_LANGUAGE > VSLANG. Returns null if none are set. + /// Returns null if none are set. private static CultureInfo GetExternalOverriddenUILanguage() { // DOTNET_CLI_UI_LANGUAGE= is the main way for users to customize the CLI's UI language via the .NET SDK. @@ -1755,18 +1757,6 @@ private static CultureInfo GetExternalOverriddenUILanguage() catch (CultureNotFoundException) { } } - // VSLANG= is set by Visual Studio. - string vsLang = Environment.GetEnvironmentVariable("VSLANG"); - if (vsLang != null && int.TryParse(vsLang, out int vsLcid)) - { - try - { - return new CultureInfo(vsLcid); - } - catch (ArgumentOutOfRangeException) { } - catch (CultureNotFoundException) { } - } - return null; } From 60ee6bccb2974e082b2f3d6a385e811bd29eb0f6 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Tue, 14 Mar 2023 09:29:35 -0700 Subject: [PATCH 17/28] Make the encoding change happen earlier than msbuild use server stuff --- src/MSBuild/XMake.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index bf311e39537..17ce67916a7 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -234,6 +234,10 @@ string[] args DumpCounters(true /* initialize only */); } + // Setup the console UI. + using AutomaticEncodingRestorer _ = new(); + SetConsoleUI(); + int exitCode; if ( ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_4) && @@ -665,11 +669,6 @@ public static ExitType Execute( // check the operating system the code is running on VerifyThrowSupportedOS(); - // Setup the console UI. - using AutomaticEncodingRestorer discardedEncodingRestorer = new(); - CultureInfo originalThreadCulture = Thread.CurrentThread.CurrentUICulture; - SetConsoleUI(); - // reset the application state for this new build ResetBuildState(); From 992fb7e6c6f4f7eb6cd23570b11db09b3c27b785 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Tue, 14 Mar 2023 09:48:58 -0700 Subject: [PATCH 18/28] Code clean up for dotnet cli language feature --- src/MSBuild.UnitTests/XMake_Tests.cs | 21 ++++++++++++++------- src/MSBuild/AutomaticEncodingRestorer.cs | 6 ++++-- src/MSBuild/MSBuild.csproj | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 736c7d920f1..f3ae4a58ab3 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -9,8 +9,8 @@ using System.IO.Compression; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices; using System.Threading; -using Newtonsoft.Json.Linq; using Microsoft.Build.CommandLine; using Microsoft.Build.Framework; using Microsoft.Build.Logging; @@ -20,7 +20,6 @@ using Shouldly; using Xunit; using Xunit.Abstractions; -using System.Runtime.InteropServices; #nullable disable @@ -648,6 +647,11 @@ public void SetConsoleUICulture() [Fact] public void ConsoleUIRespectsSDKLanguage() { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !MSBuildApp.CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()) + { + return; // The feature to detect .NET SDK Languages is not enabled on this machine, so don't test it. + } + const string DOTNET_CLI_UI_LANGUAGE = nameof(DOTNET_CLI_UI_LANGUAGE); using TestEnvironment testEnvironment = TestEnvironment.Create(); // Save the current environment info so it can be restored. @@ -665,16 +669,16 @@ public void ConsoleUIRespectsSDKLanguage() MSBuildApp.SetConsoleUI(); - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || MSBuildApp.CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()) - { - Assert.Equal(new CultureInfo("ja"), thisThread.CurrentUICulture); - Assert.Equal(65001, Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. - } + Assert.Equal(new CultureInfo("ja"), thisThread.CurrentUICulture); + Assert.Equal(65001, Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. } finally { // Restore the current UI culture back to the way it was at the beginning of this unit test. thisThread.CurrentUICulture = originalUICulture; + // Restore for full framework + CultureInfo.CurrentCulture = originalUICulture; + CultureInfo.DefaultThreadCurrentUICulture = originalUICulture; // MSbuild should also restore the encoding upon exit, but we don't create that context here. Console.OutputEncoding = originalOutputEncoding; @@ -840,6 +844,7 @@ public void MSBuildEngineLogger() { using TestEnvironment testEnvironment = TestEnvironment.Create(); testEnvironment.SetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE", "en"); // build machines may have other values. + CultureInfo.CurrentUICulture = new CultureInfo("en"); // Validate that the thread will produce an english log regardless of the machine OS language string oldValueForMSBuildLoadMicrosoftTargetsReadOnly = Environment.GetEnvironmentVariable("MSBuildLoadMicrosoftTargetsReadOnly"); string projectString = @@ -877,6 +882,8 @@ public void MSBuildEngineLogger() var logFileContents = File.ReadAllText(logFile); + Assert.Equal(new CultureInfo("en"), Thread.CurrentThread.CurrentUICulture); + logFileContents.ShouldContain("Process = "); logFileContents.ShouldContain("MSBuild executable path = "); logFileContents.ShouldContain("Command line arguments = "); diff --git a/src/MSBuild/AutomaticEncodingRestorer.cs b/src/MSBuild/AutomaticEncodingRestorer.cs index ba4acfda90b..1129cca0708 100644 --- a/src/MSBuild/AutomaticEncodingRestorer.cs +++ b/src/MSBuild/AutomaticEncodingRestorer.cs @@ -75,9 +75,11 @@ public void Dispose() /// /// Return whether the running OS is windows for net472. - /// RuntimeInformation.IsOSPlatform(OSPlatform.Windows) is supposed to be available in net472 but apparently it isnt part of full framework so we can't use it. + /// RuntimeInformation.IsOSPlatform(OSPlatform.Windows) is sometimes available in net472 but in this context it's an unavailable API, so this function is needed. /// - /// + /// + /// A boolean of 'true' iff the current machine os is windows. + /// private bool IsWindowsOS() { string? windir = Environment.GetEnvironmentVariable("windir"); diff --git a/src/MSBuild/MSBuild.csproj b/src/MSBuild/MSBuild.csproj index 039fe2e452a..00bbd4e0bf4 100644 --- a/src/MSBuild/MSBuild.csproj +++ b/src/MSBuild/MSBuild.csproj @@ -48,8 +48,8 @@ true false - full + full $(DefineConstants);MSBUILDENTRYPOINTEXE From 048bd20ddf68d838203e6f0b72c05ff3e3423c1e Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 15 Mar 2023 09:47:48 -0700 Subject: [PATCH 19/28] Respond to PR feedback --- src/MSBuild/AutomaticEncodingRestorer.cs | 47 +++++++----------------- src/MSBuild/XMake.cs | 1 + 2 files changed, 14 insertions(+), 34 deletions(-) diff --git a/src/MSBuild/AutomaticEncodingRestorer.cs b/src/MSBuild/AutomaticEncodingRestorer.cs index 1129cca0708..b5696d62ab8 100644 --- a/src/MSBuild/AutomaticEncodingRestorer.cs +++ b/src/MSBuild/AutomaticEncodingRestorer.cs @@ -20,33 +20,25 @@ public class AutomaticEncodingRestorer : IDisposable private Encoding? _originalOutputEncoding = null; private Encoding? _originalInputEncoding = null; - private bool outputEncodingAccessible = false; - private bool inputEncodingAccessible = false; - public AutomaticEncodingRestorer() { try { - if ( #if NET7_0_OR_GREATER - !OperatingSystem.IsIOS() && !OperatingSystem.IsAndroid() && !OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs. -#else - IsWindowsOS()) // Windows is the only platform where we need to change the encoding as other platforms are UTF 8 by default, so for now its the only one required to restore. -#endif + if (OperatingSystem.IsIOS() || OperatingSystem.IsAndroid() || OperatingSystem.IsTvOS()) // Output + Input Encoding are unavailable on these platforms per docs, and they're only available past net 5. { - _originalOutputEncoding = Console.OutputEncoding; - outputEncodingAccessible = true; - if ( -#if NET7_0_OR_GREATER - !OperatingSystem.IsBrowser()) // Input Encoding is also unavailable in this platform. -#else - IsWindowsOS()) + return; + } #endif - { - _originalInputEncoding = Console.InputEncoding; - inputEncodingAccessible = true; - } + _originalOutputEncoding = Console.OutputEncoding; + +#if NET7_0_OR_GREATER + if (OperatingSystem.IsBrowser()) // Input Encoding is also unavailable in this platform. (No concern for net472 as browser is unavailable.) + { + return; } +#endif + _originalInputEncoding = Console.InputEncoding; } catch (Exception ex) when (ex is IOException || ex is SecurityException) { @@ -58,11 +50,11 @@ public void Dispose() { try { - if (outputEncodingAccessible && _originalOutputEncoding != null) + if (_originalOutputEncoding != null) { Console.OutputEncoding = _originalOutputEncoding; } - if (inputEncodingAccessible && _originalInputEncoding != null) + if (_originalInputEncoding != null) { Console.InputEncoding = _originalInputEncoding; } @@ -72,18 +64,5 @@ public void Dispose() // The encoding is unavailable. Do nothing. } } - - /// - /// Return whether the running OS is windows for net472. - /// RuntimeInformation.IsOSPlatform(OSPlatform.Windows) is sometimes available in net472 but in this context it's an unavailable API, so this function is needed. - /// - /// - /// A boolean of 'true' iff the current machine os is windows. - /// - private bool IsWindowsOS() - { - string? windir = Environment.GetEnvironmentVariable("windir"); - return !string.IsNullOrEmpty(windir) && windir.Contains(@"\") && Directory.Exists(windir); - } } } diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 17ce67916a7..1f457069c18 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1639,6 +1639,7 @@ internal static void SetConsoleUI() // Eliminate the complex script cultures from the language selection. var desiredCulture = GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); thisThread.CurrentUICulture = desiredCulture; + // For full framework, both the above and below must be set. This is not true in core, but it is a no op in core. // https://learn.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.defaultthreadcurrentculture?redirectedfrom=MSDN&view=net-7.0#remarks CultureInfo.CurrentUICulture = desiredCulture; From 18f34d5ae7a75b531055c1ebc6bbb46b90e47446 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 23 Mar 2023 13:55:56 -0700 Subject: [PATCH 20/28] Follow the Microsoft style of using the string alias over System.String Co-authored-by: Rainer Sigwald --- src/MSBuild/XMake.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 1f457069c18..d117cafd687 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1734,7 +1734,7 @@ public static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() private static bool ForceUniversalEncodingOptInEnabled() { - return String.Equals(Environment.GetEnvironmentVariable("DOTNET_CLI_FORCE_UTF8_ENCODING"), "true", StringComparison.OrdinalIgnoreCase); + return string.Equals(Environment.GetEnvironmentVariable("DOTNET_CLI_FORCE_UTF8_ENCODING"), "true", StringComparison.OrdinalIgnoreCase); } /// From 881305ca8cb028e15d56c30a7ad3ce895c408f9c Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 4 May 2023 11:22:04 -0700 Subject: [PATCH 21/28] prepare to add encoding change to logs as well --- src/Build/Logging/FileLogger.cs | 5 +++++ src/Deprecated/Engine/Logging/FileLogger.cs | 4 ---- src/MSBuild/XMake.cs | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Build/Logging/FileLogger.cs b/src/Build/Logging/FileLogger.cs index 156ee0c58e5..050ede8b31d 100644 --- a/src/Build/Logging/FileLogger.cs +++ b/src/Build/Logging/FileLogger.cs @@ -50,6 +50,11 @@ public FileLogger() /// Available events. public override void Initialize(IEventSource eventSource) { + if (EncodingUtilities.GetExternalOverridenUILanguageIfSupportableWithEncoding() != null) + { + _encoding = Encoding.UTF8; + } + ErrorUtilities.VerifyThrowArgumentNull(eventSource, nameof(eventSource)); eventSource.BuildFinished += FileLoggerBuildFinished; InitializeFileLogger(eventSource, 1); diff --git a/src/Deprecated/Engine/Logging/FileLogger.cs b/src/Deprecated/Engine/Logging/FileLogger.cs index f7fd9fdf988..1f574b8af5e 100644 --- a/src/Deprecated/Engine/Logging/FileLogger.cs +++ b/src/Deprecated/Engine/Logging/FileLogger.cs @@ -117,7 +117,6 @@ public override void Initialize(IEventSource eventSource, int nodeCount) /// /// The handler for the write delegate of the console logger we are deriving from. /// - /// KieranMo /// The text to write to the log private void Write(string text) { @@ -143,7 +142,6 @@ private void Write(string text) /// /// Shutdown method implementation of ILogger - we need to flush and close our logfile. /// - /// KieranMo public override void Shutdown() { fileWriter?.Close(); @@ -152,7 +150,6 @@ public override void Shutdown() /// /// Parses out the logger parameters from the Parameters string. /// - /// KieranMo private void ParseFileLoggerParameters() { if (this.Parameters != null) @@ -180,7 +177,6 @@ private void ParseFileLoggerParameters() /// /// Apply a parameter parsed by the file logger. /// - /// KieranMo private void ApplyFileLoggerParameter(string parameterName, string parameterValue) { switch (parameterName.ToUpperInvariant()) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index d117cafd687..30526f4edfd 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -35,6 +35,7 @@ using ForwardingLoggerRecord = Microsoft.Build.Logging.ForwardingLoggerRecord; using BinaryLogger = Microsoft.Build.Logging.BinaryLogger; using Microsoft.Build.Shared.Debugging; +using Microsoft.Build.Shared.Debugging; using Microsoft.Build.Experimental; using Microsoft.Build.Framework.Telemetry; using Microsoft.Build.Internal; @@ -1687,7 +1688,7 @@ internal static void SetConsoleUI() /// The custom language that was set by the user for an 'external' tool besides MSBuild. /// Returns if none are set. /// - private static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding() + public static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding() { CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage(); if (externalLanguageSetting != null) From 45961ed782130fb056108b0c83b3e8ef399049e2 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 4 May 2023 15:25:11 -0700 Subject: [PATCH 22/28] Get the encoding to change log files as well and move the encoding up before a debugger is attached so that build errors wont have a bad encoding --- .../DistributedFileLogger.cs | 1 + src/Build/Logging/FileLogger.cs | 10 +- src/Framework/EncodingUtilities.cs | 87 ++++++++++++++++ src/MSBuild.UnitTests/XMake_Tests.cs | 2 +- src/MSBuild/XMake.cs | 98 ++----------------- 5 files changed, 101 insertions(+), 97 deletions(-) diff --git a/src/Build/Logging/DistributedLoggers/DistributedFileLogger.cs b/src/Build/Logging/DistributedLoggers/DistributedFileLogger.cs index 4df6b2830f9..7f90d035fd6 100644 --- a/src/Build/Logging/DistributedLoggers/DistributedFileLogger.cs +++ b/src/Build/Logging/DistributedLoggers/DistributedFileLogger.cs @@ -97,6 +97,7 @@ public void Initialize(IEventSource eventSource) ErrorUtilities.VerifyThrowArgumentNull(eventSource, nameof(eventSource)); ParseFileLoggerParameters(); string fileName = _logFile; + try { // Create a new file logger and pass it some parameters to make the build log very detailed diff --git a/src/Build/Logging/FileLogger.cs b/src/Build/Logging/FileLogger.cs index 050ede8b31d..96527226118 100644 --- a/src/Build/Logging/FileLogger.cs +++ b/src/Build/Logging/FileLogger.cs @@ -39,6 +39,11 @@ public FileLogger() colorReset: BaseConsoleLogger.DontResetColor) { WriteHandler = Write; + + if (EncodingUtilities.GetExternalOverridenUILanguageIfSupportableWithEncoding() != null) + { + _encoding = Encoding.UTF8; + } } #endregion @@ -50,11 +55,6 @@ public FileLogger() /// Available events. public override void Initialize(IEventSource eventSource) { - if (EncodingUtilities.GetExternalOverridenUILanguageIfSupportableWithEncoding() != null) - { - _encoding = Encoding.UTF8; - } - ErrorUtilities.VerifyThrowArgumentNull(eventSource, nameof(eventSource)); eventSource.BuildFinished += FileLoggerBuildFinished; InitializeFileLogger(eventSource, 1); diff --git a/src/Framework/EncodingUtilities.cs b/src/Framework/EncodingUtilities.cs index 9ad987bd730..67fdcdd971c 100644 --- a/src/Framework/EncodingUtilities.cs +++ b/src/Framework/EncodingUtilities.cs @@ -3,11 +3,15 @@ using System; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; +using System.Runtime.InteropServices; +using System.Security; using System.Text; using Microsoft.Build.Framework; +using Microsoft.Win32; #nullable disable @@ -247,5 +251,88 @@ internal static Encoding BatchFileEncoding(string contents, string encodingSpeci : EncodingUtilities.Utf8WithoutBom; } } + + /// + /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect the SDK variable. + /// To use the correspoding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases. + /// This code introduces a breaking change in .NET 8 due to the encoding of the console being changed. + /// If the environment variables are undefined, this function should be a no-op. + /// + /// + /// The custom language that was set by the user for an 'external' tool besides MSBuild. + /// Returns if none are set. + /// + public static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding() + { + CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage(); + if (externalLanguageSetting != null) + { + if ( + !externalLanguageSetting.TwoLetterISOLanguageName.Equals("en", StringComparison.InvariantCultureIgnoreCase) && + CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() + ) + { + // Setting both encodings causes a change in the CHCP, making it so we dont need to P-Invoke chcp ourselves. + Console.OutputEncoding = Encoding.UTF8; + // If the InputEncoding is not set, the encoding will work in CMD but not in Powershell, as the raw CHCP page won't be changed. + Console.InputEncoding = Encoding.UTF8; + return externalLanguageSetting; + } + else if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return externalLanguageSetting; + } + } + return null; + } + + public static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version.Major >= 10) // UTF-8 is only officially supported on 10+. + { + try + { + using RegistryKey windowsVersionRegistry = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion"); + var buildNumber = windowsVersionRegistry.GetValue("CurrentBuildNumber").ToString(); + const int buildNumberThatOfficialySupportsUTF8 = 18363; + return int.Parse(buildNumber) >= buildNumberThatOfficialySupportsUTF8 || ForceUniversalEncodingOptInEnabled(); + } + catch (Exception ex) when (ex is SecurityException || ex is ObjectDisposedException) + { + // We don't want to break those in VS on older versions of Windows with a non-en language. + // Allow those without registry permissions to force the encoding, however. + return ForceUniversalEncodingOptInEnabled(); + } + } + return false; + } + + private static bool ForceUniversalEncodingOptInEnabled() + { + return string.Equals(Environment.GetEnvironmentVariable("DOTNET_CLI_FORCE_UTF8_ENCODING"), "true", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE.) + /// Does NOT check System Locale or OS Display Language. + /// Ported from the .NET SDK: https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs + /// + /// The custom language that was set by the user for an 'external' tool besides MSBuild. + /// Returns null if none are set. + private static CultureInfo GetExternalOverriddenUILanguage() + { + // DOTNET_CLI_UI_LANGUAGE= is the main way for users to customize the CLI's UI language via the .NET SDK. + string dotnetCliLanguage = Environment.GetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE"); + if (dotnetCliLanguage != null) + { + try + { + return new CultureInfo(dotnetCliLanguage); + } + catch (CultureNotFoundException) { } + } + + return null; + } } } diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index f3ae4a58ab3..6c4784bed37 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -647,7 +647,7 @@ public void SetConsoleUICulture() [Fact] public void ConsoleUIRespectsSDKLanguage() { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !MSBuildApp.CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !EncodingUtilities.CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()) { return; // The feature to detect .NET SDK Languages is not enabled on this machine, so don't test it. } diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 91e97d0a5d6..3663ba46305 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -36,7 +36,6 @@ using BinaryLogger = Microsoft.Build.Logging.BinaryLogger; using LiveLogger = Microsoft.Build.Logging.LiveLogger.LiveLogger; using Microsoft.Build.Shared.Debugging; -using Microsoft.Build.Shared.Debugging; using Microsoft.Build.Experimental; using Microsoft.Build.Framework.Telemetry; using Microsoft.Build.Internal; @@ -224,6 +223,10 @@ string[] args ) #pragma warning restore SA1111, SA1009 // Closing parenthesis should be on line of last parameter { + // Setup the console UI. + using AutomaticEncodingRestorer _ = new(); + SetConsoleUI(); + DebuggerLaunchCheck(); // Initialize new build telemetry and record start of this build. @@ -236,10 +239,6 @@ string[] args DumpCounters(true /* initialize only */); } - // Setup the console UI. - using AutomaticEncodingRestorer _ = new(); - SetConsoleUI(); - int exitCode; if ( ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_4) && @@ -1276,7 +1275,7 @@ internal static bool BuildProject( } else { - Project project = projectCollection.LoadProject(projectFile, globalProperties, toolsVersion); + Evaluation.Project project = projectCollection.LoadProject(projectFile, globalProperties, toolsVersion); project.SaveLogicalProject(preprocessWriter); @@ -1522,7 +1521,7 @@ private static bool PrintTargets(string projectFile, string toolsVersion, Dictio { try { - Project project = projectCollection.LoadProject(projectFile, globalProperties, toolsVersion); + Evaluation.Project project = projectCollection.LoadProject(projectFile, globalProperties, toolsVersion); foreach (string target in project.Targets.Keys) { @@ -1677,7 +1676,7 @@ internal static void SetConsoleUI() Thread thisThread = Thread.CurrentThread; // Eliminate the complex script cultures from the language selection. - var desiredCulture = GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); + var desiredCulture = EncodingUtilities.GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); thisThread.CurrentUICulture = desiredCulture; // For full framework, both the above and below must be set. This is not true in core, but it is a no op in core. @@ -1717,89 +1716,6 @@ internal static void SetConsoleUI() #endif } - /// - /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect the SDK variable. - /// To use the correspoding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases. - /// This code introduces a breaking change due to the encoding of the console being changed. - /// If the environment variables are undefined, this function should be a no-op. - /// - /// - /// The custom language that was set by the user for an 'external' tool besides MSBuild. - /// Returns if none are set. - /// - public static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding() - { - CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage(); - if (externalLanguageSetting != null) - { - if ( - !externalLanguageSetting.TwoLetterISOLanguageName.Equals("en", StringComparison.InvariantCultureIgnoreCase) && - CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() - ) - { - // Setting both encodings causes a change in the CHCP, making it so we dont need to P-Invoke ourselves. - Console.OutputEncoding = Encoding.UTF8; - // If the InputEncoding is not set, the encoding will work in CMD but not in Powershell, as the raw CHCP page won't be changed. - Console.InputEncoding = Encoding.UTF8; - return externalLanguageSetting; - } - else if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return externalLanguageSetting; - } - } - return null; - } - - public static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version.Major >= 10) // UTF-8 is only officially supported on 10+. - { - try - { - using RegistryKey windowsVersionRegistry = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion"); - var buildNumber = windowsVersionRegistry.GetValue("CurrentBuildNumber").ToString(); - const int buildNumberThatOfficialySupportsUTF8 = 18363; - return int.Parse(buildNumber) >= buildNumberThatOfficialySupportsUTF8 || ForceUniversalEncodingOptInEnabled(); - } - catch (Exception ex) when (ex is SecurityException || ex is ObjectDisposedException) - { - // We don't want to break those in VS on older versions of Windows with a non-en language. - // Allow those without registry permissions to force the encoding, however. - return ForceUniversalEncodingOptInEnabled(); - } - } - return false; - } - - private static bool ForceUniversalEncodingOptInEnabled() - { - return string.Equals(Environment.GetEnvironmentVariable("DOTNET_CLI_FORCE_UTF8_ENCODING"), "true", StringComparison.OrdinalIgnoreCase); - } - - /// - /// Look at UI language overrides that can be set by known external invokers. (DOTNET_CLI_UI_LANGUAGE.) - /// Does NOT check System Locale or OS Display Language. - /// Ported from the .NET SDK: https://github.com/dotnet/sdk/blob/bcea1face15458814b8e53e8785b52ba464f6538/src/Cli/Microsoft.DotNet.Cli.Utils/UILanguageOverride.cs - /// - /// The custom language that was set by the user for an 'external' tool besides MSBuild. - /// Returns null if none are set. - private static CultureInfo GetExternalOverriddenUILanguage() - { - // DOTNET_CLI_UI_LANGUAGE= is the main way for users to customize the CLI's UI language via the .NET SDK. - string dotnetCliLanguage = Environment.GetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE"); - if (dotnetCliLanguage != null) - { - try - { - return new CultureInfo(dotnetCliLanguage); - } - catch (CultureNotFoundException) { } - } - - return null; - } - /// /// Gets all specified switches, from the command line, as well as all /// response files, including the auto-response file. From 41d9da63b236d7b4f3b70377200048c6f493cb76 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Tue, 9 May 2023 09:48:06 -0700 Subject: [PATCH 23/28] Add environment variable to disable/opt-out of the new feature --- src/Build/Logging/FileLogger.cs | 2 +- src/Framework/EncodingUtilities.cs | 7 ++++++- src/MSBuild/XMake.cs | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Build/Logging/FileLogger.cs b/src/Build/Logging/FileLogger.cs index 96527226118..180d58a2a08 100644 --- a/src/Build/Logging/FileLogger.cs +++ b/src/Build/Logging/FileLogger.cs @@ -40,7 +40,7 @@ public FileLogger() { WriteHandler = Write; - if (EncodingUtilities.GetExternalOverridenUILanguageIfSupportableWithEncoding() != null) + if (EncodingUtilities.GetExternalOverriddenUILanguageIfSupportableWithEncoding() != null) { _encoding = Encoding.UTF8; } diff --git a/src/Framework/EncodingUtilities.cs b/src/Framework/EncodingUtilities.cs index 67fdcdd971c..20475ca62b7 100644 --- a/src/Framework/EncodingUtilities.cs +++ b/src/Framework/EncodingUtilities.cs @@ -262,8 +262,13 @@ internal static Encoding BatchFileEncoding(string contents, string encodingSpeci /// The custom language that was set by the user for an 'external' tool besides MSBuild. /// Returns if none are set. /// - public static CultureInfo GetExternalOverridenUILanguageIfSupportableWithEncoding() + public static CultureInfo GetExternalOverriddenUILanguageIfSupportableWithEncoding() { + if (string.Equals(Environment.GetEnvironmentVariable("MSBUILDDISABLEDYNAMICUTFENCODING"), "true", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage(); if (externalLanguageSetting != null) { diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 3663ba46305..15ccab46b32 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1676,7 +1676,7 @@ internal static void SetConsoleUI() Thread thisThread = Thread.CurrentThread; // Eliminate the complex script cultures from the language selection. - var desiredCulture = EncodingUtilities.GetExternalOverridenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); + var desiredCulture = EncodingUtilities.GetExternalOverriddenUILanguageIfSupportableWithEncoding() ?? CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(); thisThread.CurrentUICulture = desiredCulture; // For full framework, both the above and below must be set. This is not true in core, but it is a no op in core. From aa5a55b1fdd9988437c8ad437d5075f09877e83e Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Tue, 9 May 2023 13:47:05 -0700 Subject: [PATCH 24/28] Move the utf8 encoding change feature flag to the change wave --- src/Framework/EncodingUtilities.cs | 9 +++++---- src/MSBuild.UnitTests/XMake_Tests.cs | 25 ++++++++++++++++++------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/Framework/EncodingUtilities.cs b/src/Framework/EncodingUtilities.cs index 20475ca62b7..607b5f6a430 100644 --- a/src/Framework/EncodingUtilities.cs +++ b/src/Framework/EncodingUtilities.cs @@ -254,7 +254,7 @@ internal static Encoding BatchFileEncoding(string contents, string encodingSpeci /// /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect the SDK variable. - /// To use the correspoding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases. + /// To use the corresponding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases. /// This code introduces a breaking change in .NET 8 due to the encoding of the console being changed. /// If the environment variables are undefined, this function should be a no-op. /// @@ -264,7 +264,7 @@ internal static Encoding BatchFileEncoding(string contents, string encodingSpeci /// public static CultureInfo GetExternalOverriddenUILanguageIfSupportableWithEncoding() { - if (string.Equals(Environment.GetEnvironmentVariable("MSBUILDDISABLEDYNAMICUTFENCODING"), "true", StringComparison.OrdinalIgnoreCase)) + if (!ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_8)) { return null; } @@ -277,9 +277,9 @@ public static CultureInfo GetExternalOverriddenUILanguageIfSupportableWithEncodi CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() ) { - // Setting both encodings causes a change in the CHCP, making it so we dont need to P-Invoke chcp ourselves. + // Setting both encodings causes a change in the CHCP, making it so we don't need to P-Invoke CHCP ourselves. Console.OutputEncoding = Encoding.UTF8; - // If the InputEncoding is not set, the encoding will work in CMD but not in Powershell, as the raw CHCP page won't be changed. + // If the InputEncoding is not set, the encoding will work in CMD but not in PowerShell, as the raw CHCP page won't be changed. Console.InputEncoding = Encoding.UTF8; return externalLanguageSetting; } @@ -288,6 +288,7 @@ public static CultureInfo GetExternalOverriddenUILanguageIfSupportableWithEncodi return externalLanguageSetting; } } + return null; } diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 6c4784bed37..63544655de0 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -644,8 +644,10 @@ public void SetConsoleUICulture() } - [Fact] - public void ConsoleUIRespectsSDKLanguage() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ConsoleUIRespectsSDKLanguage(bool enableFeature) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !EncodingUtilities.CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding()) { @@ -665,12 +667,19 @@ public void ConsoleUIRespectsSDKLanguage() try { // Set the UI language based on the SDK environment var. - testEnvironment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, "ja"); // japanese chose arbitrarily. - + testEnvironment.SetEnvironmentVariable(DOTNET_CLI_UI_LANGUAGE, "ja"); // Japanese chose arbitrarily. + ChangeWaves.ResetStateForTests(); + if (!enableFeature) + { + testEnvironment.SetEnvironmentVariable("MSBUILDDISABLEFEATURESFROMVERSION", ChangeWaves.Wave17_8.ToString()); + } MSBuildApp.SetConsoleUI(); - Assert.Equal(new CultureInfo("ja"), thisThread.CurrentUICulture); - Assert.Equal(65001, Console.OutputEncoding.CodePage); // utf 8 enabled for correct rendering. + Assert.Equal(enableFeature ? new CultureInfo("ja") : originalUICulture, thisThread.CurrentUICulture); + if (enableFeature) + { + Assert.Equal(65001, Console.OutputEncoding.CodePage); // UTF-8 enabled for correct rendering. + } } finally { @@ -680,9 +689,11 @@ public void ConsoleUIRespectsSDKLanguage() CultureInfo.CurrentCulture = originalUICulture; CultureInfo.DefaultThreadCurrentUICulture = originalUICulture; - // MSbuild should also restore the encoding upon exit, but we don't create that context here. + // MSBuild should also restore the encoding upon exit, but we don't create that context here. Console.OutputEncoding = originalOutputEncoding; Console.InputEncoding = originalInputEncoding; + + BuildEnvironmentHelper.ResetInstance_ForUnitTestsOnly(); } } From 5c2db937ab0824443566473fe80776c2a48ad2c5 Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Tue, 9 May 2023 16:39:23 -0500 Subject: [PATCH 25/28] Remove some no-longer-required usings --- src/MSBuild/XMake.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 15ccab46b32..3e7a6f19c74 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -39,9 +39,6 @@ using Microsoft.Build.Experimental; using Microsoft.Build.Framework.Telemetry; using Microsoft.Build.Internal; -using Microsoft.Build.Logging.LiveLogger; -using System.Runtime.InteropServices; -using Microsoft.Win32; #nullable disable @@ -1275,7 +1272,7 @@ internal static bool BuildProject( } else { - Evaluation.Project project = projectCollection.LoadProject(projectFile, globalProperties, toolsVersion); + Project project = projectCollection.LoadProject(projectFile, globalProperties, toolsVersion); project.SaveLogicalProject(preprocessWriter); @@ -1521,7 +1518,7 @@ private static bool PrintTargets(string projectFile, string toolsVersion, Dictio { try { - Evaluation.Project project = projectCollection.LoadProject(projectFile, globalProperties, toolsVersion); + Project project = projectCollection.LoadProject(projectFile, globalProperties, toolsVersion); foreach (string target in project.Targets.Keys) { From 28b8bd056b703141c66362cad7c58dce5089b508 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Tue, 9 May 2023 14:53:07 -0700 Subject: [PATCH 26/28] Remove the english language and redirect in the help link Co-authored-by: Forgind <12969783+Forgind@users.noreply.github.com> --- src/MSBuild/XMake.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 3e7a6f19c74..428397f1f7c 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1677,7 +1677,7 @@ internal static void SetConsoleUI() thisThread.CurrentUICulture = desiredCulture; // For full framework, both the above and below must be set. This is not true in core, but it is a no op in core. - // https://learn.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.defaultthreadcurrentculture?redirectedfrom=MSDN&view=net-7.0#remarks + // https://learn.microsoft.com/dotnet/api/system.globalization.cultureinfo.defaultthreadcurrentculture#remarks CultureInfo.CurrentUICulture = desiredCulture; CultureInfo.DefaultThreadCurrentUICulture = desiredCulture; From e61c559b8f4af45b014215911ed8f6614aa7d9d4 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Tue, 9 May 2023 15:40:07 -0700 Subject: [PATCH 27/28] Consider that a user may delete windows registry keys --- src/Framework/EncodingUtilities.cs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Framework/EncodingUtilities.cs b/src/Framework/EncodingUtilities.cs index 607b5f6a430..298c740da96 100644 --- a/src/Framework/EncodingUtilities.cs +++ b/src/Framework/EncodingUtilities.cs @@ -251,7 +251,7 @@ internal static Encoding BatchFileEncoding(string contents, string encodingSpeci : EncodingUtilities.Utf8WithoutBom; } } - +#nullable enable /// /// The .NET SDK and Visual Studio both have environment variables that set a custom language. MSBuild should respect the SDK variable. /// To use the corresponding UI culture, in certain cases the console encoding must be changed. This function will change the encoding in these cases. @@ -262,14 +262,14 @@ internal static Encoding BatchFileEncoding(string contents, string encodingSpeci /// The custom language that was set by the user for an 'external' tool besides MSBuild. /// Returns if none are set. /// - public static CultureInfo GetExternalOverriddenUILanguageIfSupportableWithEncoding() + public static CultureInfo? GetExternalOverriddenUILanguageIfSupportableWithEncoding() { if (!ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_8)) { return null; } - CultureInfo externalLanguageSetting = GetExternalOverriddenUILanguage(); + CultureInfo? externalLanguageSetting = GetExternalOverriddenUILanguage(); if (externalLanguageSetting != null) { if ( @@ -298,18 +298,19 @@ public static bool CurrentPlatformIsWindowsAndOfficiallySupportsUTF8Encoding() { try { - using RegistryKey windowsVersionRegistry = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion"); - var buildNumber = windowsVersionRegistry.GetValue("CurrentBuildNumber").ToString(); - const int buildNumberThatOfficialySupportsUTF8 = 18363; - return int.Parse(buildNumber) >= buildNumberThatOfficialySupportsUTF8 || ForceUniversalEncodingOptInEnabled(); + using RegistryKey? windowsVersionRegistry = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion"); + string? buildNumber = windowsVersionRegistry?.GetValue("CurrentBuildNumber")?.ToString(); + const int buildNumberThatOfficiallySupportsUTF8 = 18363; + return buildNumber != null && (int.Parse(buildNumber) >= buildNumberThatOfficiallySupportsUTF8 || ForceUniversalEncodingOptInEnabled()); } - catch (Exception ex) when (ex is SecurityException || ex is ObjectDisposedException) + catch (Exception ex) when (ex is SecurityException or ObjectDisposedException) { // We don't want to break those in VS on older versions of Windows with a non-en language. // Allow those without registry permissions to force the encoding, however. return ForceUniversalEncodingOptInEnabled(); } } + return false; } @@ -325,10 +326,10 @@ private static bool ForceUniversalEncodingOptInEnabled() /// /// The custom language that was set by the user for an 'external' tool besides MSBuild. /// Returns null if none are set. - private static CultureInfo GetExternalOverriddenUILanguage() + private static CultureInfo? GetExternalOverriddenUILanguage() { // DOTNET_CLI_UI_LANGUAGE= is the main way for users to customize the CLI's UI language via the .NET SDK. - string dotnetCliLanguage = Environment.GetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE"); + string? dotnetCliLanguage = Environment.GetEnvironmentVariable("DOTNET_CLI_UI_LANGUAGE"); if (dotnetCliLanguage != null) { try @@ -342,3 +343,4 @@ private static CultureInfo GetExternalOverriddenUILanguage() } } } + From f7946399a67c57130916f9ba9ede8d09c2e0d39e Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 10 May 2023 11:32:13 -0700 Subject: [PATCH 28/28] update test to expect the fallback culture instead of the original culture as the existing code could change the culture to the fallback culture instead of the original culture --- newc/Program.cs | 2 ++ newc/newc.csproj | 10 ++++++++++ src/MSBuild.UnitTests/XMake_Tests.cs | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 newc/Program.cs create mode 100644 newc/newc.csproj diff --git a/newc/Program.cs b/newc/Program.cs new file mode 100644 index 00000000000..3751555cbd3 --- /dev/null +++ b/newc/Program.cs @@ -0,0 +1,2 @@ +// See https://aka.ms/new-console-template for more information +Console.WriteLine("Hello, World!"); diff --git a/newc/newc.csproj b/newc/newc.csproj new file mode 100644 index 00000000000..2150e3797ba --- /dev/null +++ b/newc/newc.csproj @@ -0,0 +1,10 @@ + + + + Exe + net8.0 + enable + enable + + + diff --git a/src/MSBuild.UnitTests/XMake_Tests.cs b/src/MSBuild.UnitTests/XMake_Tests.cs index 63544655de0..083782e52c7 100644 --- a/src/MSBuild.UnitTests/XMake_Tests.cs +++ b/src/MSBuild.UnitTests/XMake_Tests.cs @@ -675,7 +675,7 @@ public void ConsoleUIRespectsSDKLanguage(bool enableFeature) } MSBuildApp.SetConsoleUI(); - Assert.Equal(enableFeature ? new CultureInfo("ja") : originalUICulture, thisThread.CurrentUICulture); + Assert.Equal(enableFeature ? new CultureInfo("ja") : CultureInfo.CurrentUICulture.GetConsoleFallbackUICulture(), thisThread.CurrentUICulture); if (enableFeature) { Assert.Equal(65001, Console.OutputEncoding.CodePage); // UTF-8 enabled for correct rendering.