Make ALL template warnings show up as test errors (not just C# warnin…

…gs), and produce more diagnostic artifacts (#17637)

- Ensure all template build warnings are caught as errors and make the tests fail
- Save template test output as artifacts even if they pass

src/TestUtils/src/Microsoft.Maui.IntegrationTests/BaseBuildTest.cs

@@ -98,25 +98,10 @@ public void BuildTestSetUp()
 		[TearDown]
 		public void BuildTestTearDown()
 		{
-			// Clean up test or attach content from failed tests
-			if (TestContext.CurrentContext.Result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Passed ||
-				TestContext.CurrentContext.Result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Skipped)
+			// Attach test content and logs as artifacts
+			foreach (var log in Directory.GetFiles(Path.Combine(TestDirectory), "*log", SearchOption.AllDirectories))
 			{
-				try
-				{
-					if (Directory.Exists(TestDirectory))
-						Directory.Delete(TestDirectory, recursive: true);
-				}
-				catch (IOException)
-				{
-				}
-			}
-			else
-			{
-				foreach (var log in Directory.GetFiles(Path.Combine(TestDirectory), "*log", SearchOption.AllDirectories))
-				{
-					TestContext.AddTestAttachment(log, Path.GetFileName(TestDirectory));
-				}
+				TestContext.AddTestAttachment(log, Path.GetFileName(TestDirectory));
 			}
 		}
 

src/TestUtils/src/Microsoft.Maui.IntegrationTests/TemplateTests.cs

@@ -43,7 +43,7 @@ public void Build(string id, string framework, string config, bool shouldPack)
 					"<PropertyGroup><Version>1.0.0-preview.1</Version></PropertyGroup></Project>");
 
 			string target = shouldPack ? "Pack" : "";
-			Assert.IsTrue(DotnetInternal.Build(projectFile, config, target: target, properties: BuildProps),
+			Assert.IsTrue(DotnetInternal.Build(projectFile, config, target: target, properties: BuildProps, msbuildWarningsAsErrors: true),
 				$"Project {Path.GetFileName(projectFile)} failed to build. Check test output/attachments for errors.");
 		}
 
@@ -69,7 +69,7 @@ public void BuildUnpackaged(string id, string framework, string config)
 				"<UseMaui>true</UseMaui>",
 				"<UseMaui>true</UseMaui><WindowsPackageType>None</WindowsPackageType>");
 
-			Assert.IsTrue(DotnetInternal.Build(projectFile, config, properties: BuildProps),
+			Assert.IsTrue(DotnetInternal.Build(projectFile, config, properties: BuildProps, msbuildWarningsAsErrors: true),
 				$"Project {Path.GetFileName(projectFile)} failed to build. Check test output/attachments for errors.");
 		}
 
@@ -107,6 +107,9 @@ void AssetExists(string filename)
 			}
 		}
 
+		/// <summary>
+		/// Tests the scenario where a .NET MAUI Library specifically uses UseMauiCore instead of UseMaui.
+		/// </summary>
 		[Test]
 		[TestCase("mauilib", DotNetPrevious, "Debug")]
 		[TestCase("mauilib", DotNetPrevious, "Release")]
@@ -121,14 +124,23 @@ public void PackCoreLib(string id, string framework, string config)
 				$"Unable to create template {id}. Check test output for errors.");
 
 			EnableTizen(projectFile);
-			FileUtilities.ReplaceInFile(projectFile, new Dictionary<string, string>()
+
+			var projectSectionsToReplace = new Dictionary<string, string>()
 			{
-				{ "UseMaui", "UseMauiCore" },
+				{ "UseMaui", "UseMauiCore" }, // This is the key part of the test
 				{ "SingleProject", "EnablePreviewMsixTooling" },
-			});
+			};
+			if (framework != "net6.0")
+			{
+				// On versions after net6.0 this package reference also has to be updated to ensure the version of the MAUI Core package
+				// is specified and avoids the MA002 warning.
+				projectSectionsToReplace.Add("Include=\"Microsoft.Maui.Controls\"", "Include=\"Microsoft.Maui.Core\"");
+			}
+
+			FileUtilities.ReplaceInFile(projectFile, projectSectionsToReplace);
 			Directory.Delete(Path.Combine(projectDir, "Platforms"), recursive: true);
 
-			Assert.IsTrue(DotnetInternal.Build(projectFile, config, properties: BuildProps),
+			Assert.IsTrue(DotnetInternal.Build(projectFile, config, properties: BuildProps, msbuildWarningsAsErrors: true),
 				$"Project {Path.GetFileName(projectFile)} failed to build. Check test output/attachments for errors.");
 		}
 
@@ -152,7 +164,7 @@ public void BuildWithoutPackageReference(string id, string framework, string con
 				"<PackageReference Include=\"Microsoft.Maui.Controls\" Version=\"$(MauiVersion)\" />",
 				"");
 
-			Assert.IsTrue(DotnetInternal.Build(projectFile, config, properties: BuildProps),
+			Assert.IsTrue(DotnetInternal.Build(projectFile, config, properties: BuildProps, msbuildWarningsAsErrors: true),
 				$"Project {Path.GetFileName(projectFile)} failed to build. Check test output/attachments for errors.");
 		}
 
@@ -179,7 +191,7 @@ public void BuildWithDifferentVersionNumber(string id, string config, string dis
 				$"<ApplicationVersion>1</ApplicationVersion>",
 				$"<ApplicationVersion>{version}</ApplicationVersion>");
 
-			Assert.IsTrue(DotnetInternal.Build(projectFile, config, properties: BuildProps),
+			Assert.IsTrue(DotnetInternal.Build(projectFile, config, properties: BuildProps, msbuildWarningsAsErrors: true),
 				$"Project {Path.GetFileName(projectFile)} failed to build. Check test output/attachments for errors.");
 		}
 
@@ -205,7 +217,7 @@ public void CheckEntitlementsForMauiBlazorOnMacCatalyst(string id, string config
 			};
 
 			Assert.IsTrue(DotnetInternal.New(id, projectDir, framework), $"Unable to create template {id}. Check test output for errors.");
-			Assert.IsTrue(DotnetInternal.Build(projectFile, config, framework: $"{framework}-maccatalyst", properties: buildWithCodeSignProps),
+			Assert.IsTrue(DotnetInternal.Build(projectFile, config, framework: $"{framework}-maccatalyst", properties: buildWithCodeSignProps, msbuildWarningsAsErrors: true),
 				$"Project {Path.GetFileName(projectFile)} failed to build. Check test output/attachments for errors.");
 
 			List<string> expectedEntitlements = config == "Release" ?

src/TestUtils/src/Microsoft.Maui.IntegrationTests/Utilities/DotnetInternal.cs

@@ -7,7 +7,7 @@ public static class DotnetInternal
 		static readonly string DotnetTool = Path.Combine(TestEnvironment.GetMauiDirectory(), "bin", "dotnet", "dotnet");
 		const int DEFAULT_TIMEOUT = 900;
 
-		public static bool Build(string projectFile, string config, string target = "", string framework = "", IEnumerable<string>? properties = null, string binlogPath = "")
+		public static bool Build(string projectFile, string config, string target = "", string framework = "", IEnumerable<string>? properties = null, string binlogPath = "", bool msbuildWarningsAsErrors = false)
 		{
 			var binlogName = $"build-{DateTime.UtcNow.ToFileTimeUtc()}.binlog";
 			var buildArgs = $"\"{projectFile}\" -c {config}";
@@ -34,6 +34,26 @@ public static bool Build(string projectFile, string config, string target = "",
 				binlogPath = Path.Combine(Path.GetDirectoryName(projectFile) ?? "", binlogName);
 			}
 
+			if (msbuildWarningsAsErrors)
+			{
+				// We set WarnAsError to specifically cause *MSBuild* warnings to be errors (setting TreatWarningsAsErrors
+				// affect only C# compiler warnings).
+				buildArgs += " -warnaserror";
+
+				// However, we need to ignore specific MSBuild warnings that are acceptable in these tests:
+				var csWarningsToIgnore = new string[]
+				{
+					"NETSDK1201", // Details: "For projects targeting .NET 8.0 and higher, specifying a RuntimeIdentifier
+								// will no longer produce a self contained app by default. To continue building
+								// self-contained apps, set the SelfContained property to true or use the --self-contained
+								// argument."
+								// Justification: This warning isn't meaningful in this test scenario.
+					"CS1591", // Details: "Missing XML comment for publicly visible type or member 'XYZ'"
+							// Justification: It's OK for templates to have missing doc comments.
+				};
+				buildArgs += " " + string.Join(" ", csWarningsToIgnore.Select(csWarning => $"-p:nowarn={csWarning}"));
+			}
+
 			return Run("build", $"{buildArgs} -bl:\"{binlogPath}\"");
 		}
 
@@ -82,8 +102,10 @@ public static bool New(string shortName, string outputDirectory, string framewor
 		public static bool Run(string command, string args, int timeoutinSeconds = DEFAULT_TIMEOUT)
 		{
 			var runOutput = RunForOutput(command, args, out int exitCode, timeoutinSeconds);
-			if (exitCode != 0)
-				TestContext.WriteLine(runOutput);
+			TestContext.WriteLine($"Process exit code: {exitCode}");
+			TestContext.WriteLine($"-------- Process output start --------");
+			TestContext.WriteLine(runOutput);
+			TestContext.WriteLine($"-------- Process output end --------");
 
 			return exitCode == 0;
 		}