Implement validation and fix abandoned validation mutex

Fixes wixtoolset/issues#5946
Fixes wixtoolset/issues#6366

src/WixToolset.Core.Native/IWindowsInstallerValidatorCallback.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixToolset.Core.Native
+{
+    /// <summary>
+    /// Callbacks during validation.
+    /// </summary>
+    public interface IWindowsInstallerValidatorCallback
+    {
+        /// <summary>
+        /// Indicates if the validator callback encountered an error.
+        /// </summary>
+        bool EncounteredError { get; }
+
+        /// <summary>
+        /// Validation blocked by another Windows Installer operation.
+        /// </summary>
+        void ValidationBlocked();
+
+        /// <summary>
+        /// Validation message from an ICE.
+        /// </summary>
+        /// <param name="message">The validation message.</param>
+        /// <returns>True if validation should continue; otherwise cancel the validation.</returns>
+        bool ValidationMessage(ValidationMessage message);
+    }
+}

src/WixToolset.Core.Native/ValidationMessage.cs

@@ -0,0 +1,47 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixToolset.Core.Native
+{
+    using System.Collections.Generic;
+
+    /// <summary>
+    /// Message from ICE
+    /// </summary>
+    public class ValidationMessage
+    {
+        /// <summary>
+        /// Name of the ICE providing the message.
+        /// </summary>
+        public string IceName { get; set; }
+
+        /// <summary>
+        /// Validation type.
+        /// </summary>
+        public ValidationMessageType Type { get; set; }
+
+        /// <summary>
+        /// Message text.
+        /// </summary>
+        public string Description { get; set; }
+
+        /// <summary>
+        /// Optional help URL for the message.
+        /// </summary>
+        public string HelpUrl { get; set; }
+
+        /// <summary>
+        /// Optional table causing the message.
+        /// </summary>
+        public string Table { get; set; }
+
+        /// <summary>
+        /// Optional column causing the message.
+        /// </summary>
+        public string Column { get; set; }
+
+        /// <summary>
+        /// Optional primary keys causing the message.
+        /// </summary>
+        public IEnumerable<string> PrimaryKeys { get; set; }
+    }
+}

src/WixToolset.Core.Native/ValidationMessageType.cs

@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
+
+namespace WixToolset.Core.Native
+{
+    /// <summary>
+    /// Validation message type.
+    /// </summary>
+    public enum ValidationMessageType
+    {
+        /// <summary>
+        /// Failure message reporting the failure of the ICE custom action.
+        /// </summary>
+        InternalFailure = 0,
+
+        /// <summary>
+        /// Error message reporting database authoring that case incorrect behavior.
+        /// </summary>
+        Error = 1,
+
+        /// <summary>
+        /// Warning message reporting database authoring that causes incorrect behavior in certain cases.
+        /// Warnings can also report unexpected side-effects of database authoring.
+        /// </summary>
+        Warning = 2,
+
+        /// <summary>
+        /// Informational message.
+        /// </summary>
+        Info = 3,
+    };
+}