[Codegen] Allow any nullable double fields to be strings or doubles a…

…nd improved error reporting (#984)

* Allow any nullable double fields to be strings or doubles and improved error reporting

* small fix

src/HassModel/NetDaemon.HassModel.CodeGenerator/Controller.cs

@@ -23,10 +23,10 @@ public Controller(CodeGenerationSettings generationSettings, HomeAssistantSettin
     private string EntityMetaDataFileName => Path.Combine(OutputFolder, "EntityMetaData.json");
     private string ServicesMetaDataFileName => Path.Combine(OutputFolder, "ServicesMetaData.json");
 
-    private string OutputFolder => string.IsNullOrEmpty(_generationSettings.OutputFolder) 
-        ? Directory.GetParent(Path.GetFullPath(_generationSettings.OutputFile))!.FullName 
+    private string OutputFolder => string.IsNullOrEmpty(_generationSettings.OutputFolder)
+        ? Directory.GetParent(Path.GetFullPath(_generationSettings.OutputFile))!.FullName
         : _generationSettings.OutputFolder;
-    
+
     public async Task RunAsync()
     {
         var (hassStates, servicesMetaData) = await HaRepositry.GetHaData(_haSettings).ConfigureAwait(false);
@@ -38,11 +38,35 @@ public async Task RunAsync()
         await Save(mergedEntityMetaData, EntityMetaDataFileName).ConfigureAwait(false);
         await Save(servicesMetaData, ServicesMetaDataFileName).ConfigureAwait(false);
 
-        var generatedTypes = Generator.GenerateTypes(mergedEntityMetaData.Domains, ServiceMetaDataParser.Parse(servicesMetaData!.Value));
+        var hassServiceDomains = ServiceMetaDataParser.Parse(servicesMetaData!.Value, out var deserializationErrors);
+        CheckParseErrors(deserializationErrors);
+
+        var generatedTypes = Generator.GenerateTypes(mergedEntityMetaData.Domains, hassServiceDomains);
 
         SaveGeneratedCode(generatedTypes);
     }
 
+    internal static void CheckParseErrors(List<DeserializationError> parseErrors)
+    {
+        if (parseErrors.Count == 0) return;
+
+        Console.ForegroundColor = ConsoleColor.Yellow;
+        Console.WriteLine("""
+                          Errors occured while parsing metadata from Home Assistant for one or more services.
+                          This is usually caused by metadata from HA that is not in the expected JSON format.
+                          nd-codegen will try to continue to generate code for other services.
+                          """);
+        Console.ResetColor();
+        foreach (var deserializationError in parseErrors)
+        {
+            Console.WriteLine();
+            Console.WriteLine(deserializationError.Exception);
+            Console.WriteLine(deserializationError.Context + " = ");
+            Console.Out.Flush();
+            Console.WriteLine(JsonSerializer.Serialize(deserializationError.Element, new JsonSerializerOptions{WriteIndented = true}));
+        }
+    }
+
     internal async Task<EntitiesMetaData> LoadEntitiesMetaDataAsync()
     {
         var fileStream = File.Exists(EntityMetaDataFileName) switch
@@ -72,15 +96,15 @@ private async Task Save<T>(T merged, string fileName)
         await using var _ = fileStream.ConfigureAwait(false);
         await JsonSerializer.SerializeAsync(fileStream, merged, JsonSerializerOptions).ConfigureAwait(false);
     }
-    
+
     private static JsonSerializerOptions JsonSerializerOptions =>
         new()
         {
             PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
             WriteIndented = true,
             Converters = { new ClrTypeJsonConverter() }
         };
-    
+
     private void SaveGeneratedCode(MemberDeclarationSyntax[] generatedTypes)
     {
         if (!_generationSettings.GenerateOneFilePerEntity)
@@ -112,4 +136,4 @@ private void SaveGeneratedCode(MemberDeclarationSyntax[] generatedTypes)
             Console.WriteLine(OutputFolder);
         }
     }
-}
+}

src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/SelectorConverter.cs

@@ -25,7 +25,7 @@ class SelectorConverter : JsonConverter<Selector>
             return new Selector { Type = selectorName};
         }
 
-        var deserialize = (Selector?)element.Deserialize(selectorType, ServiceMetaDataParser.SnakeCaseNamingPolicySerializerOptions);
+        var deserialize = (Selector?)element.Deserialize(selectorType, ServiceMetaDataParser.SerializerOptions);
         deserialize ??= (Selector)Activator.CreateInstance(selectorType)!;
 
         return deserialize with { Type = selectorName };

src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/Selectors.cs

@@ -39,19 +39,11 @@ internal record EntitySelector : Selector
 
 internal record NumberSelector : Selector
 {
-    [Required]
-    public double Min { get; init; }
+    public double? Min { get; init; }
 
-    [Required]
-    public double Max { get; init; }
+    public double? Max { get; init; }
 
-
-    // Step can also contain the string "any" which is not usefull for our purpose, se we deserialize as a string and then try to parse as a double
-    [JsonPropertyName("step")]
-    public string? StepValue { get; init; }
-
-    [JsonIgnore]
-    public double? Step => double.TryParse(StepValue, out var d) ? d: null;
+    public double? Step { get; init; }
 
     public string? UnitOfMeasurement { get; init; }
 }

src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/ServiceMetaDataParser.cs

@@ -1,76 +1,86 @@
 namespace NetDaemon.HassModel.CodeGenerator.Model;
 
+public record DeserializationError(Exception Exception, string? Context, JsonElement Element);
+
 internal static class ServiceMetaDataParser
 {
-    public static readonly JsonSerializerOptions SnakeCaseNamingPolicySerializerOptions = new()
+
+    public static readonly JsonSerializerOptions SerializerOptions = new()
     {
-        PropertyNamingPolicy = SnakeCaseNamingPolicy.Instance
+        PropertyNamingPolicy = SnakeCaseNamingPolicy.Instance,
+        Converters = { new StringAsDoubleConverter() }
     };
 
-
     public static IReadOnlyCollection<HassServiceDomain> Parse(JsonElement element) => Parse(element, out _);
 
     /// <summary>
     ///     Parses all json elements to instance result from GetServices call
     /// </summary>
     /// <param name="element">JsonElement containing the result data</param>
     /// <param name="errors">Outputs Any Exceptions during deserialization</param>
-    public static IReadOnlyCollection<HassServiceDomain> Parse(JsonElement element, out List<Exception> errors)
+    public static IReadOnlyCollection<HassServiceDomain> Parse(JsonElement element, out List<DeserializationError> errors)
     {
-        errors = new List<Exception>();
+        errors = new List<DeserializationError>();
         if (element.ValueKind != JsonValueKind.Object)
             throw new InvalidOperationException("Not expected result from the GetServices result");
 
         var hassServiceDomains = new List<HassServiceDomain>();
-        foreach (var property in element.EnumerateObject())
+        foreach (var domainProperty in element.EnumerateObject())
         {
             try
             {
                 var hassServiceDomain = new HassServiceDomain
                 {
-                    Domain = property.Name,
-                    Services = GetServices(property.Value)
+                    Domain = domainProperty.Name,
+                    Services = GetServices(domainProperty.Value, errors, domainProperty.Name)
                 };
                 hassServiceDomains.Add(hassServiceDomain);
             }
             catch (JsonException e)
             {
-                Console.Error.WriteLine($"JSON deserialization of {nameof(HassServiceDomain)} failed: {e.Message}");
-                Console.Error.WriteLine($"Deserialization source was: {property.Value}");
-                errors.Add(e);
+                errors.Add(new (e, domainProperty.Name, domainProperty.Value));
             }
         }
         return hassServiceDomains;
     }
 
-    private static IReadOnlyCollection<HassService> GetServices(JsonElement element)
+    private static IReadOnlyCollection<HassService> GetServices(JsonElement domainElement, List<DeserializationError> errors, string context)
     {
-        return element.EnumerateObject()
+        return domainElement.EnumerateObject()
             .Select(serviceDomainProperty =>
-                GetService(serviceDomainProperty.Name, serviceDomainProperty.Value)).ToList();
-    }
+                GetService(serviceDomainProperty.Name, serviceDomainProperty.Value, errors, context))
+            .OfType<HassService>().ToList();
+  }
 
-    private static HassService GetService(string service, JsonElement element)
+    private static HassService? GetService(string serviceName, JsonElement serviceElement, List<DeserializationError> errors, string context)
     {
-        var result = element.Deserialize<HassService>(SnakeCaseNamingPolicySerializerOptions)! with
+        try
         {
-            Service = service,
-        };
-
-        if (element.TryGetProperty("fields", out var fieldProperty))
-        {
-            result = result with
+            var result = serviceElement.Deserialize<HassService>(SerializerOptions)! with
             {
-                Fields = fieldProperty.EnumerateObject().Select(p => GetField(p.Name, p.Value)).ToList()
+                Service = serviceName,
             };
-        }
 
-        return result;
+            if (serviceElement.TryGetProperty("fields", out var fieldProperty))
+            {
+                result = result with
+                {
+                    Fields = fieldProperty.EnumerateObject().Select(p => GetField(p.Name, p.Value)).ToList()
+                };
+            }
+
+            return result;
+        }
+        catch (Exception ex)
+        {
+            errors.Add(new (ex, $"{context}.{serviceName}", serviceElement));
+            return null;
+        }
     }
 
     private static HassServiceField GetField(string fieldName, JsonElement element)
     {
-        return element.Deserialize<HassServiceField>(SnakeCaseNamingPolicySerializerOptions)! with
+        return element.Deserialize<HassServiceField>(SerializerOptions)! with
         {
             Field = fieldName,
         };

src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/StringAsDoubleConverter.cs

@@ -0,0 +1,25 @@
+using System.Text.Json.Serialization;
+
+namespace NetDaemon.HassModel.CodeGenerator.Model;
+
+class StringAsDoubleConverter : JsonConverter<double?>
+{
+    public override double? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        // Some fields (step) can have a string or a numeric value. If it is a string we will try to parse it to a decimal
+        return reader.TokenType switch
+        {
+            JsonTokenType.Number => reader.GetDouble(),
+            JsonTokenType.String => double.TryParse(reader.GetString(), out var d) ? d : null,
+            _ => Skip(ref reader)
+        };
+    }
+
+    double? Skip(ref Utf8JsonReader reader)
+    {
+        reader.Skip();
+        return null;
+    }
+
+    public override void Write(Utf8JsonWriter writer, double? value, JsonSerializerOptions options) => throw new NotSupportedException();
+}

src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServiceMetaDataParserTest.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Text.Json;
+using NetDaemon.HassModel.CodeGenerator;
 using NetDaemon.HassModel.CodeGenerator.Model;
 
 namespace NetDaemon.HassModel.Tests.CodeGenerator;
@@ -143,7 +144,6 @@ public void DeserializeTargetEntityArray()
 
     }
 
-
     [Fact]
     public void NumericStepCanBeAny()
     {
@@ -183,18 +183,30 @@ public void NumericStepCanBeAny()
                                      "name":"Longitude",
                                      "description":"Longitude of your location."
                                   },
-                                  "elevation":{
-                                     "required":false,
-                                     "example":120,
-                                     "selector":{
-                                        "number":{
-                                           "mode":"box",
-                                           "step":"1"
-                                        }
-                                     },
-                                     "name":"Elevation",
-                                     "description":"Elevation of your location."
-                                  }
+                                 "elevation":{
+                                    "required":false,
+                                    "example":120,
+                                    "selector":{
+                                       "number":{
+                                          "mode":"box",
+                                          "step":"1"
+                                       }
+                                    },
+                                    "name":"Elevation",
+                                    "description":"Elevation of your location."
+                                 },
+                                 "testNumberStep":{
+                                    "required":false,
+                                    "example":120,
+                                    "selector":{
+                                       "number":{
+                                          "mode":"box",
+                                          "step":0.01
+                                       }
+                                    },
+                                    "name":"Elevation",
+                                    "description":"Elevation of your location."
+                                 }
                                }
                             }
                          }
@@ -203,7 +215,82 @@ public void NumericStepCanBeAny()
         var result = Parse(sample);
 
         var steps = result.Single().Services.Single().Fields!.Select(f => (f.Selector as NumberSelector)!.Step).ToArray();
-        steps.Should().Equal(null, null, 1); // any is mapped to null
+        steps.Should().Equal(null, null, 1, 0.01d); // any is mapped to null
+    }
+
+
+        [Fact]
+    public void JsonError()
+    {
+        var sample = """
+                     {
+                       "orbiter_services": {
+                         "invalid_json_service": {
+                           "name": "Observe Planet",
+                           "description": ["Array is not allowed here!"],
+                           "fields": {
+                             "frequency": {
+                               "required": 1.1,
+                               "example": false,
+                               "selector": {
+                                 "number": {
+                                   "multiple": false,
+                                   "mode": "box",
+                                   "min": "N/A",
+                                   "max": "N/A",
+                                   "step": "any"
+                                 }
+                               }
+                             }
+                           }
+                         },
+                         "navigate": {
+                           "name": "Navigates to a new location",
+                           "fields": {
+                             "latitude": {
+                               "required": true,
+                               "example": 32.87336,
+                               "selector": {
+                                 "number": {
+                                   "mode": 212,
+                                   "min": -90,
+                                   "max": 90,
+                                   "step": "any"
+                                 }
+                               },
+                               "name": "Latitude",
+                               "description": "Latitude of your location."
+                             },
+                             "longitude": {
+                               "required": true,
+                               "example": 117.22743,
+                               "selector": {
+                                 "number": {
+                                   "mode": "box",
+                                   "min": -180,
+                                   "max": 180,
+                                   "step": "any"
+                                 }
+                               },
+                               "name": "Longitude",
+                               "description": "Longitude of your location."
+                             }
+                           }
+                         }
+                       }
+                     }
+                     """;
+        var element = JsonDocument.Parse(sample).RootElement;
+        var result = ServiceMetaDataParser.Parse(element, out var errors);
+
+        errors.Should().HaveCount(1, because: "We should get an error for the failed service");
+        errors.Single().Context.Should().Be("orbiter_services.invalid_json_service");
+
+        result.Should().HaveCount(1, because:"The service that is valid should still be parsed ");
+        result.Single().Services.Should().HaveCount(1);
+
+        // Just to manually validate the console output while running in the in the IDE
+        Controller.CheckParseErrors(errors);
     }
 
     private static IReadOnlyCollection<HassServiceDomain> Parse(string sample)

src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServicesGeneratorTest.cs

@@ -33,7 +33,7 @@ public void TestServicesGeneration()
                         Service = "turn_on",
                         Fields = new HassServiceField[] {
                             new() { Field = "transition", Selector = new NumberSelector(), },
-                            new() { Field = "brightness", Selector = new NumberSelector { StepValue = "0.2" }, }
+                            new() { Field = "brightness", Selector = new NumberSelector { Step = 0.2d }, }
                         },
                         Target = new TargetSelector
                         {