Add Json Schema Validation for publishednodes.json file (#1305)

* Added Json schema validation of publishednodes.json, with backward compatibility and jobs deserialization bug fix.
* Updated with latest json schema version and removed any additional properties added previously in tests to conform to previous schema versions.
* Initial implementation of json-everything as replacement for Newtonsoft Json Schema (#2). Replace Newtonsoft Schema validation with JsonSchemaDotNet.
* Remove lingering references to Newtonsoft.Json.Schema library.
* Remove duplicate schema file and convert to linked asset with copy always for tests.
* Delete modules/src/Microsoft.Azure.IIoT.Modules.OpcUa.Publisher/src/schemas directory. GitHub seems to think this folder still exists though it is not in the repository anymore. Removing in an attempt to clean up the PR.
* Revert version change to sln file, address LGTM automated review feedback.
* Removed trailing whitespaces from solution by temporarily using SA1028 rule locally + added missing copyright notice.
* Second Item Array Failure Schema Fix.
* Add schema spec documentation.
* Update schema with additional comments from code review.
* Add endpoint url regex design notes to design doc.
* Addressed PR review comments.
* Correct typos across docs, schema and tests.
* Update command line doc to include post 2.8.0 additional args.
* Update Publisher* to Published* in Legacy CLI files.
* Changed casing from this -> This in schema.
* Revert heading changes in publisher cmd line docs. This change reverts the heading for the heading for the publisher cmd line args doc back to the original version per code upstream review comment.

Co-authored-by: Suneet Nangia <suneetnangia@gmail.com>

Industrial-IoT.sln

@@ -226,6 +226,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.IIoT.OpcUa.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.IIoT.Agent.Framework.Tests", "common\src\Microsoft.Azure.IIoT.Agent.Framework\tests\Microsoft.Azure.IIoT.Agent.Framework.Tests.csproj", "{63F1C41D-6D07-4F0D-967C-CBF3752139A0}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.IIoT.Validators.JsonSchemaDotNet", "common\src\Microsoft.Azure.IIoT.Validators.JsonSchemaDotNet\src\Microsoft.Azure.IIoT.Validators.JsonSchemaDotNet.csproj", "{019C942B-5FC7-4417-BE1D-40E7E6869D1B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Azure.IIoT.Validators.JsonSchemaDotNet.Tests", "common\src\Microsoft.Azure.IIoT.Validators.JsonSchemaDotNet\tests\Microsoft.Azure.IIoT.Validators.JsonSchemaDotNet.Tests.csproj", "{1C45566A-D8AC-4EAE-9735-39663784D2DF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -922,6 +926,22 @@ Global
 		{63F1C41D-6D07-4F0D-967C-CBF3752139A0}.Release|Any CPU.Build.0 = Release|Any CPU
 		{63F1C41D-6D07-4F0D-967C-CBF3752139A0}.Release|x64.ActiveCfg = Release|Any CPU
 		{63F1C41D-6D07-4F0D-967C-CBF3752139A0}.Release|x64.Build.0 = Release|Any CPU
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B}.Debug|x64.Build.0 = Debug|Any CPU
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B}.Release|x64.ActiveCfg = Release|Any CPU
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B}.Release|x64.Build.0 = Release|Any CPU
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF}.Debug|x64.Build.0 = Debug|Any CPU
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF}.Release|x64.ActiveCfg = Release|Any CPU
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1031,6 +1051,8 @@ Global
 		{422C4B02-BC9F-4494-893F-7214F8FB17E1} = {ADB416A3-C375-4362-B2C0-3A35DA4CB5F8}
 		{6F5D5CE2-57A3-4116-8A82-72120C9461C6} = {1266F300-8BD2-4F1E-BA88-911F5F3AC36C}
 		{63F1C41D-6D07-4F0D-967C-CBF3752139A0} = {BBEE26AC-8733-469B-B816-0A54EA99FC59}
+		{019C942B-5FC7-4417-BE1D-40E7E6869D1B} = {FEE95825-2E34-45AF-BDED-52F9C7412BF5}
+		{1C45566A-D8AC-4EAE-9735-39663784D2DF} = {BBEE26AC-8733-469B-B816-0A54EA99FC59}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F09EFCEA-59F6-4A16-9CB6-7E865A3F62D8}

common/src/Microsoft.Azure.IIoT.Abstractions/src/Crypto/Models/RevocationInfo.cs

@@ -12,7 +12,7 @@ namespace Microsoft.Azure.IIoT.Crypto.Models {
     public class RevocationInfo {
 
         /// <summary>
-        /// Revocation date 
+        /// Revocation date
         /// </summary>
         public DateTime? Date { get; set; }
     }

common/src/Microsoft.Azure.IIoT.Abstractions/src/Serializers/Extensions/SerializerEx.cs

@@ -135,12 +135,14 @@ public static string SerializeArrayPretty(
         /// <param name="serializer"></param>
         /// <param name="str"></param>
         /// <param name="type"></param>
+        /// <param name="schemaReader"></param>
         /// <returns></returns>
         public static object Deserialize(this ISerializer serializer,
-            string str, Type type) {
+            string str, Type type, TextReader schemaReader = null) {
             var buffer = serializer.ContentEncoding?.GetBytes(str)
                 ?? Convert.FromBase64String(str);
-            return serializer.Deserialize(buffer, type);
+
+            return serializer.Deserialize(buffer, type, schemaReader);
         }
 
         /// <summary>
@@ -167,16 +169,34 @@ public static T Deserialize<T>(this ISerializer serializer,
             return serializer.Deserialize<T>(reader.ReadToEnd());
         }
 
+        /// <summary>
+        /// Deserialize from validating reader
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="serializer"></param>
+        /// <param name="reader"></param>
+        /// <param name="schemaReader"></param>
+        /// <returns></returns>
+        public static T Deserialize<T>(this ISerializer serializer,
+            TextReader reader,
+            TextReader schemaReader) {
+
+            // Desrialize and validate json content in a single read cycle.
+            return serializer.Deserialize<T>(reader.ReadToEnd(), schemaReader);
+        }
+
         /// <summary>
         /// Deserialize from string
         /// </summary>
         /// <typeparam name="T"></typeparam>
         /// <param name="serializer"></param>
         /// <param name="json"></param>
+        /// <param name="schemaReader"></param>
         /// <returns></returns>
         public static T Deserialize<T>(this ISerializer serializer,
-            string json) {
-            var typed = serializer.Deserialize(json, typeof(T));
+            string json,
+            TextReader schemaReader = null) {
+            var typed = serializer.Deserialize(json, typeof(T), schemaReader);
             return typed == null ? default : (T)typed;
         }
 

common/src/Microsoft.Azure.IIoT.Abstractions/src/Serializers/ISerializer.cs

@@ -6,6 +6,7 @@
 namespace Microsoft.Azure.IIoT.Serializers {
     using System;
     using System.Buffers;
+    using System.IO;
     using System.Text;
 
     /// <summary>
@@ -38,8 +39,9 @@ void Serialize(IBufferWriter<byte> buffer, object o,
         /// </summary>
         /// <param name="type"></param>
         /// <param name="buffer"></param>
+        /// <param name="schemaReader"></param>
         /// <returns></returns>
-        object Deserialize(ReadOnlyMemory<byte> buffer, Type type);
+        object Deserialize(ReadOnlyMemory<byte> buffer, Type type, TextReader schemaReader = null);
 
         /// <summary>
         /// Deserialize to variant value

common/src/Microsoft.Azure.IIoT.Abstractions/src/Storage/Models/DatabaseOptions.cs

@@ -11,7 +11,7 @@ namespace Microsoft.Azure.IIoT.Storage {
     public class DatabaseOptions {
 
         /// <summary>
-        /// Database Consistency 
+        /// Database Consistency
         /// </summary>
         public OperationConsistency? Consistency { get; set; }
     }

common/src/Microsoft.Azure.IIoT.Abstractions/src/Validators/IJsonSchemaValidator.cs

@@ -0,0 +1,22 @@
+// ------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation.  All rights reserved.
+//  Licensed under the MIT License (MIT). See License.txt in the repo root for license information.
+// ------------------------------------------------------------
+
+namespace Microsoft.Azure.IIoT.Validators {
+    using System.Collections.Generic;
+    using System.IO;
+
+    /// <summary>
+    /// Json schema validator
+    /// </summary>
+    public interface IJsonSchemaValidator {
+        /// <summary>
+        /// Validates Json against the provided Json schema.
+        /// </summary>
+        /// <param name="jsonBuffer"></param>
+        /// <param name="schemaReader"></param>
+        /// <returns></returns>
+        public IList<JsonSchemaValidationResult> Validate(byte[] jsonBuffer, TextReader schemaReader);
+    }
+}

common/src/Microsoft.Azure.IIoT.Abstractions/src/Validators/JsonSchemaValidationResult.cs

@@ -0,0 +1,50 @@
+// ------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation.  All rights reserved.
+//  Licensed under the MIT License (MIT). See License.txt in the repo root for license information.
+// ------------------------------------------------------------
+
+namespace Microsoft.Azure.IIoT.Validators {
+    /// <summary>
+    /// Json schema validation result.
+    /// </summary>
+    public class JsonSchemaValidationResult {
+
+        /// <summary>
+        /// Instantiates json schema validation result.
+        /// </summary>
+        /// <param name="isValid"></param>
+        /// <param name="message"></param>
+        /// <param name="schemaLocation"></param>
+        /// <param name="instanceLocation"></param>
+        public JsonSchemaValidationResult(
+            bool isValid,
+            string message = null,
+            string schemaLocation = null,
+            string instanceLocation = null) {
+            IsValid = isValid;
+            Message = message;
+            SchemaLocation = schemaLocation;
+            InstanceLocation = instanceLocation;
+        }
+
+        /// <summary>
+        /// Indicates whether the validation passed or failed.
+        /// </summary>
+        public bool IsValid { get; }
+
+        /// <summary>
+        /// The error message, if any.
+        /// </summary>
+        public string Message { get; }
+
+        /// <summary>
+        /// The schema location that generated this node.
+        /// </summary>
+        public string SchemaLocation { get; }
+
+        /// <summary>
+        /// The instance location that was processed.
+        /// </summary>
+        public string InstanceLocation { get; }
+    }
+}

common/src/Microsoft.Azure.IIoT.Agent.Framework/src/Agent/Default/WorkerSupervisor.cs

@@ -73,7 +73,7 @@ private Task<IWorker> CreateWorker() {
             if (_instances.Count >= maxWorkers) {
                 throw new MaxWorkersReachedException(maxWorkers);
             }
-            
+
             var childScope = _lifetimeScope.BeginLifetimeScope();
             var worker = childScope.Resolve<IWorker>(new NamedParameter("workerInstance", _instances.Count));
             _instances[worker] = childScope;
@@ -87,11 +87,11 @@ private Task<IWorker> CreateWorker() {
         /// </summary>
         /// <returns>awaitable task</returns>
         private async Task StopWorker() {
-            // sort workers, so that a worker in state Stopped, Stopping or WaitingForJob will terminate first 
+            // sort workers, so that a worker in state Stopped, Stopping or WaitingForJob will terminate first
             var worker = _instances.OrderBy(kvp => kvp.Key.Status).First();
             var workerId = worker.Key.WorkerId;
             _logger.Information("Stopping worker with id {WorkerId}", workerId);
-            
+
             await worker.Key.StopAsync();
             worker.Value.Dispose();
             _instances.Remove(worker.Key);

common/src/Microsoft.Azure.IIoT.Agent.Framework/tests/Worker/WorkerSupervisorTests.cs

@@ -88,7 +88,7 @@ public async Task Test_001_DefaultConfiguration_Expect_FiveWorker() {
 
         [Fact]
         public async Task Test_002_IncreaseMaxWorker_Expect_WorkerSpawned() {
-            
+
             using var container = GetAutofacTestConfiguration();
             var agentConfig = new TestAgentConfigProvider();
             var loggerMock = new Mock<ILogger>();
@@ -140,7 +140,7 @@ public async Task Test_004_NegativeMaxWorker_Expect_UseDefaultValue() {
             var agentConfig = new TestAgentConfigProvider();
             var loggerMock = new Mock<ILogger>();
             loggerMock.Setup(l => l.Error(
-                It.Is<string>(s => s.Contains("MaxWorker")), 
+                It.Is<string>(s => s.Contains("MaxWorker")),
                 It.Is<int>(i => i == kDefaultMaxWorker)))
                 .Verifiable();
 
@@ -155,7 +155,7 @@ public async Task Test_004_NegativeMaxWorker_Expect_UseDefaultValue() {
             const int numberOfWorker = -4;
             agentConfig.SetMaxWorker(numberOfWorker);
 
-            await Task.Delay(kDefaultDelay); 
+            await Task.Delay(kDefaultDelay);
             Assert.Equal(kDefaultMaxWorker, sut.NumberOfWorkers);
             loggerMock.Verify();
 

common/src/Microsoft.Azure.IIoT.Core/src/Diagnostics/Default/EventSourceBroker.cs

@@ -37,7 +37,7 @@ public IDisposable Subscribe(string eventSource, IEventSourceSubscriber subscrib
             if (string.IsNullOrEmpty(eventSource)) {
                 throw new ArgumentNullException(nameof(eventSource));
             }
-            var source = _subscribers.GetOrAdd(eventSource, 
+            var source = _subscribers.GetOrAdd(eventSource,
                 name => new EventSourceWrapper(this));
             return source.Add(subscriber);
         }
@@ -81,7 +81,7 @@ class EventSourceWrapper {
             /// </summary>
             /// <param name="listener"></param>
             /// <param name="eventSource"></param>
-            internal EventSourceWrapper(EventSourceBroker listener, 
+            internal EventSourceWrapper(EventSourceBroker listener,
                 EventSource eventSource = null) {
                 EventSource = eventSource;
                 _listener = listener;
@@ -103,7 +103,7 @@ internal void OnEvent(EventWrittenEventArgs eventData) {
             internal void UpdateEventLevel() {
                 if (EventSource != null) {
                     lock (_subscriptions) {
-                        var level = _subscriptions.Any() ? 
+                        var level = _subscriptions.Any() ?
                             _subscriptions.Max(s => s.Level) : EventLevel.LogAlways;
                         if (level != _enabledLevel) {
                             _enabledLevel = level;
@@ -151,7 +151,7 @@ internal void Remove(EventSourceSubscriber eventSourceSubscriber) {
 
             private EventLevel _enabledLevel = EventLevel.LogAlways;
             private readonly EventSourceBroker _listener;
-            private readonly List<EventSourceSubscriber> _subscriptions = 
+            private readonly List<EventSourceSubscriber> _subscriptions =
                 new List<EventSourceSubscriber>();
         }
 

common/src/Microsoft.Azure.IIoT.Core/src/Extensions/ByteArrayEx.cs

@@ -75,7 +75,7 @@ public static string ToBase64String(this byte[] value) {
         /// </summary>
         /// <param name="bytestr">string to hash</param>
         /// <returns></returns>
-        [Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5350:Do Not Use Weak Cryptographic Algorithms", 
+        [Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5350:Do Not Use Weak Cryptographic Algorithms",
             Justification = "SHA1 not used for crypto operation.")]
         public static string ToSha1Hash(this byte[] bytestr) {
             if (bytestr == null) {

common/src/Microsoft.Azure.IIoT.Core/src/Hub/Client/IoTHubServiceHttpClient.cs

@@ -90,8 +90,8 @@ public Task<DeviceTwinModel> CreateOrUpdateAsync(DeviceTwinModel twin, bool forc
                     // Continue onward
                     // Update the deviceScope if the twin provided is for leaf iot device
                     //  (not iotedge device or iotedge module)
-                    if (!(twin.Capabilities?.IotEdge).GetValueOrDefault(false) && 
-                        string.IsNullOrEmpty(twin.ModuleId) && 
+                    if (!(twin.Capabilities?.IotEdge).GetValueOrDefault(false) &&
+                        string.IsNullOrEmpty(twin.ModuleId) &&
                         !string.IsNullOrEmpty(twin.DeviceScope)) {
                         try {
                             var update = NewRequest($"/devices/{twin.Id}");

common/src/Microsoft.Azure.IIoT.Core/src/Hub/IIoTHubDeviceTwinEventHandler.cs

@@ -14,7 +14,7 @@ public interface IIoTHubDeviceTwinEventHandler {
 
         /// <summary>
         /// Handles twin change events. Each handler is
-        /// called in sequence.  Since the event is 
+        /// called in sequence.  Since the event is
         /// mutable the next handler can process an
         /// updated version of it.
         /// </summary>

common/src/Microsoft.Azure.IIoT.Core/src/Net/Models/PortRange.cs

@@ -176,7 +176,7 @@ public static IEnumerable<PortRange> Parse(string value) {
                         highsInt = int.Parse(highs);
                     }
 
-                    if (lowsInt < IPEndPoint.MinPort || 
+                    if (lowsInt < IPEndPoint.MinPort ||
                         highsInt > IPEndPoint.MaxPort ||
                         lowsInt > highsInt){
                         throw new Exception("Port numbers are out of the range");

common/src/Microsoft.Azure.IIoT.Diagnostics.AppInsights/src/Extensions/ContainerBuilderEx.cs

@@ -79,7 +79,7 @@ public static ContainerBuilder AddDependencyTracking(this ContainerBuilder build
             depModule.ExcludeComponentCorrelationHttpHeadersOnDomains.Add("localhost");
             depModule.ExcludeComponentCorrelationHttpHeadersOnDomains.Add("127.0.0.1");
 
-            // Enable known dependency tracking, note that in future versions, we will extend this list. 
+            // Enable known dependency tracking, note that in future versions, we will extend this list.
             // Please check default settings in https://github.com/microsoft/ApplicationInsights-dotnet-server/blob/develop/WEB/Src/DependencyCollector/DependencyCollector/ApplicationInsights.config.install.xdt
 
             depModule.IncludeDiagnosticSourceActivities.Add("Microsoft.Azure.ServiceBus");

common/src/Microsoft.Azure.IIoT.Hub.Module.Client/src/Default/IoTSdkFactory.cs

@@ -142,7 +142,7 @@ public void Dispose() {
 
 
         /// <inheritdoc/>
-        [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5359:Do Not Disable Certificate Validation", 
+        [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5359:Do Not Disable Certificate Validation",
             Justification = "<Pending>")]
         public async Task<IClient> CreateAsync(string product, IProcessControl ctrl) {
 

common/src/Microsoft.Azure.IIoT.Hub.Module.Client/src/Extensions/MetricServerEx.cs

@@ -26,7 +26,7 @@ public static void StartWhenEnabled(this IMetricServer server, IModuleConfig con
                     logger.Error(e, "Unable to start metric server. For more info, please check troubleshooting guide for edge metrics collection");
                 }
 
-            } 
+            }
             else {
                 logger.Information("Metrics Collection is disabled. Not starting prometheus metric server.");
             }

common/src/Microsoft.Azure.IIoT.Hub.Module.Framework/src/Hosting/SettingsRouter.cs

@@ -67,7 +67,7 @@ public async Task<IDictionary<string, VariantValue>> ProcessSettingsAsync(
                         }
                     }
                     catch (Exception ex) {
-                        _logger.Error(ex, "Error processing setting {key}/{value}", 
+                        _logger.Error(ex, "Error processing setting {key}/{value}",
                             setting.Key, setting.Value.ToJson());
                     }
                 }

common/src/Microsoft.Azure.IIoT.Serializers.MessagePack/src/Default/MessagePackSerializer.cs

@@ -23,6 +23,7 @@ namespace Microsoft.Azure.IIoT.Serializers.MessagePack {
     using MsgPackWriter = global::MessagePack.MessagePackBinary;
     using MessagePackSerializationException = System.Exception;
     using MessagePackSerializerOptions = global::MessagePack.IFormatterResolver;
+    using System.IO;
 #endif
 
     /// <summary>
@@ -83,7 +84,7 @@ public MessagePackSerializer(
         }
 
         /// <inheritdoc/>
-        public object Deserialize(ReadOnlyMemory<byte> buffer, Type type) {
+        public object Deserialize(ReadOnlyMemory<byte> buffer, Type type, TextReader schemaReader = null) {
             try {
 #if MessagePack2
                 return MsgPack.Deserialize(type, buffer, Options);