From f8080a954caee35c300290dab7948effccccd583 Mon Sep 17 00:00:00 2001 From: Frank Bakker Date: Fri, 3 Nov 2023 07:25:00 +0100 Subject: [PATCH] [Codegen] Allow any nullable double fields to be strings or doubles and improved error reporting (#984) * Allow any nullable double fields to be strings or doubles and improved error reporting * small fix --- .../Controller.cs | 38 ++++-- .../ServicesMetaData/SelectorConverter.cs | 2 +- .../MetaData/ServicesMetaData/Selectors.cs | 14 +-- .../ServicesMetaData/ServiceMetaDataParser.cs | 64 ++++++---- .../StringAsDoubleConverter.cs | 25 ++++ .../ServiceMetaDataParserTest.cs | 115 +++++++++++++++--- .../CodeGenerator/ServicesGeneratorTest.cs | 2 +- 7 files changed, 199 insertions(+), 61 deletions(-) create mode 100644 src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/StringAsDoubleConverter.cs diff --git a/src/HassModel/NetDaemon.HassModel.CodeGenerator/Controller.cs b/src/HassModel/NetDaemon.HassModel.CodeGenerator/Controller.cs index 2c833e4f1..6fcf274a5 100644 --- a/src/HassModel/NetDaemon.HassModel.CodeGenerator/Controller.cs +++ b/src/HassModel/NetDaemon.HassModel.CodeGenerator/Controller.cs @@ -23,10 +23,10 @@ public Controller(CodeGenerationSettings generationSettings, HomeAssistantSettin private string EntityMetaDataFileName => Path.Combine(OutputFolder, "EntityMetaData.json"); private string ServicesMetaDataFileName => Path.Combine(OutputFolder, "ServicesMetaData.json"); - private string OutputFolder => string.IsNullOrEmpty(_generationSettings.OutputFolder) - ? Directory.GetParent(Path.GetFullPath(_generationSettings.OutputFile))!.FullName + private string OutputFolder => string.IsNullOrEmpty(_generationSettings.OutputFolder) + ? Directory.GetParent(Path.GetFullPath(_generationSettings.OutputFile))!.FullName : _generationSettings.OutputFolder; - + public async Task RunAsync() { var (hassStates, servicesMetaData) = await HaRepositry.GetHaData(_haSettings).ConfigureAwait(false); @@ -38,11 +38,35 @@ public async Task RunAsync() await Save(mergedEntityMetaData, EntityMetaDataFileName).ConfigureAwait(false); await Save(servicesMetaData, ServicesMetaDataFileName).ConfigureAwait(false); - var generatedTypes = Generator.GenerateTypes(mergedEntityMetaData.Domains, ServiceMetaDataParser.Parse(servicesMetaData!.Value)); + var hassServiceDomains = ServiceMetaDataParser.Parse(servicesMetaData!.Value, out var deserializationErrors); + CheckParseErrors(deserializationErrors); + + var generatedTypes = Generator.GenerateTypes(mergedEntityMetaData.Domains, hassServiceDomains); SaveGeneratedCode(generatedTypes); } + internal static void CheckParseErrors(List parseErrors) + { + if (parseErrors.Count == 0) return; + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine(""" + Errors occured while parsing metadata from Home Assistant for one or more services. + This is usually caused by metadata from HA that is not in the expected JSON format. + nd-codegen will try to continue to generate code for other services. + """); + Console.ResetColor(); + foreach (var deserializationError in parseErrors) + { + Console.WriteLine(); + Console.WriteLine(deserializationError.Exception); + Console.WriteLine(deserializationError.Context + " = "); + Console.Out.Flush(); + Console.WriteLine(JsonSerializer.Serialize(deserializationError.Element, new JsonSerializerOptions{WriteIndented = true})); + } + } + internal async Task LoadEntitiesMetaDataAsync() { var fileStream = File.Exists(EntityMetaDataFileName) switch @@ -72,7 +96,7 @@ private async Task Save(T merged, string fileName) await using var _ = fileStream.ConfigureAwait(false); await JsonSerializer.SerializeAsync(fileStream, merged, JsonSerializerOptions).ConfigureAwait(false); } - + private static JsonSerializerOptions JsonSerializerOptions => new() { @@ -80,7 +104,7 @@ private async Task Save(T merged, string fileName) WriteIndented = true, Converters = { new ClrTypeJsonConverter() } }; - + private void SaveGeneratedCode(MemberDeclarationSyntax[] generatedTypes) { if (!_generationSettings.GenerateOneFilePerEntity) @@ -112,4 +136,4 @@ private void SaveGeneratedCode(MemberDeclarationSyntax[] generatedTypes) Console.WriteLine(OutputFolder); } } -} \ No newline at end of file +} diff --git a/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/SelectorConverter.cs b/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/SelectorConverter.cs index baa5a6e2f..30cdadefa 100644 --- a/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/SelectorConverter.cs +++ b/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/SelectorConverter.cs @@ -25,7 +25,7 @@ class SelectorConverter : JsonConverter return new Selector { Type = selectorName}; } - var deserialize = (Selector?)element.Deserialize(selectorType, ServiceMetaDataParser.SnakeCaseNamingPolicySerializerOptions); + var deserialize = (Selector?)element.Deserialize(selectorType, ServiceMetaDataParser.SerializerOptions); deserialize ??= (Selector)Activator.CreateInstance(selectorType)!; return deserialize with { Type = selectorName }; diff --git a/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/Selectors.cs b/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/Selectors.cs index 50982fa9e..bb755cf42 100644 --- a/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/Selectors.cs +++ b/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/Selectors.cs @@ -39,19 +39,11 @@ internal record EntitySelector : Selector internal record NumberSelector : Selector { - [Required] - public double Min { get; init; } + public double? Min { get; init; } - [Required] - public double Max { get; init; } + public double? Max { get; init; } - - // Step can also contain the string "any" which is not usefull for our purpose, se we deserialize as a string and then try to parse as a double - [JsonPropertyName("step")] - public string? StepValue { get; init; } - - [JsonIgnore] - public double? Step => double.TryParse(StepValue, out var d) ? d: null; + public double? Step { get; init; } public string? UnitOfMeasurement { get; init; } } diff --git a/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/ServiceMetaDataParser.cs b/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/ServiceMetaDataParser.cs index 255046f1a..a61bd0344 100644 --- a/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/ServiceMetaDataParser.cs +++ b/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/ServiceMetaDataParser.cs @@ -1,13 +1,16 @@ namespace NetDaemon.HassModel.CodeGenerator.Model; +public record DeserializationError(Exception Exception, string? Context, JsonElement Element); + internal static class ServiceMetaDataParser { - public static readonly JsonSerializerOptions SnakeCaseNamingPolicySerializerOptions = new() + + public static readonly JsonSerializerOptions SerializerOptions = new() { - PropertyNamingPolicy = SnakeCaseNamingPolicy.Instance + PropertyNamingPolicy = SnakeCaseNamingPolicy.Instance, + Converters = { new StringAsDoubleConverter() } }; - public static IReadOnlyCollection Parse(JsonElement element) => Parse(element, out _); /// @@ -15,62 +18,69 @@ internal static class ServiceMetaDataParser /// /// JsonElement containing the result data /// Outputs Any Exceptions during deserialization - public static IReadOnlyCollection Parse(JsonElement element, out List errors) + public static IReadOnlyCollection Parse(JsonElement element, out List errors) { - errors = new List(); + errors = new List(); if (element.ValueKind != JsonValueKind.Object) throw new InvalidOperationException("Not expected result from the GetServices result"); var hassServiceDomains = new List(); - foreach (var property in element.EnumerateObject()) + foreach (var domainProperty in element.EnumerateObject()) { try { var hassServiceDomain = new HassServiceDomain { - Domain = property.Name, - Services = GetServices(property.Value) + Domain = domainProperty.Name, + Services = GetServices(domainProperty.Value, errors, domainProperty.Name) }; hassServiceDomains.Add(hassServiceDomain); } catch (JsonException e) { - Console.Error.WriteLine($"JSON deserialization of {nameof(HassServiceDomain)} failed: {e.Message}"); - Console.Error.WriteLine($"Deserialization source was: {property.Value}"); - errors.Add(e); + errors.Add(new (e, domainProperty.Name, domainProperty.Value)); } } return hassServiceDomains; } - private static IReadOnlyCollection GetServices(JsonElement element) + private static IReadOnlyCollection GetServices(JsonElement domainElement, List errors, string context) { - return element.EnumerateObject() + return domainElement.EnumerateObject() .Select(serviceDomainProperty => - GetService(serviceDomainProperty.Name, serviceDomainProperty.Value)).ToList(); - } + GetService(serviceDomainProperty.Name, serviceDomainProperty.Value, errors, context)) + .OfType().ToList(); + } - private static HassService GetService(string service, JsonElement element) + private static HassService? GetService(string serviceName, JsonElement serviceElement, List errors, string context) { - var result = element.Deserialize(SnakeCaseNamingPolicySerializerOptions)! with + try { - Service = service, - }; - - if (element.TryGetProperty("fields", out var fieldProperty)) - { - result = result with + var result = serviceElement.Deserialize(SerializerOptions)! with { - Fields = fieldProperty.EnumerateObject().Select(p => GetField(p.Name, p.Value)).ToList() + Service = serviceName, }; - } - return result; + if (serviceElement.TryGetProperty("fields", out var fieldProperty)) + { + result = result with + { + Fields = fieldProperty.EnumerateObject().Select(p => GetField(p.Name, p.Value)).ToList() + }; + } + + return result; + } + catch (Exception ex) + { + errors.Add(new (ex, $"{context}.{serviceName}", serviceElement)); + return null; + } } private static HassServiceField GetField(string fieldName, JsonElement element) { - return element.Deserialize(SnakeCaseNamingPolicySerializerOptions)! with + return element.Deserialize(SerializerOptions)! with { Field = fieldName, }; diff --git a/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/StringAsDoubleConverter.cs b/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/StringAsDoubleConverter.cs new file mode 100644 index 000000000..3cac5bca9 --- /dev/null +++ b/src/HassModel/NetDaemon.HassModel.CodeGenerator/MetaData/ServicesMetaData/StringAsDoubleConverter.cs @@ -0,0 +1,25 @@ +using System.Text.Json.Serialization; + +namespace NetDaemon.HassModel.CodeGenerator.Model; + +class StringAsDoubleConverter : JsonConverter +{ + public override double? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + // Some fields (step) can have a string or a numeric value. If it is a string we will try to parse it to a decimal + return reader.TokenType switch + { + JsonTokenType.Number => reader.GetDouble(), + JsonTokenType.String => double.TryParse(reader.GetString(), out var d) ? d : null, + _ => Skip(ref reader) + }; + } + + double? Skip(ref Utf8JsonReader reader) + { + reader.Skip(); + return null; + } + + public override void Write(Utf8JsonWriter writer, double? value, JsonSerializerOptions options) => throw new NotSupportedException(); +} diff --git a/src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServiceMetaDataParserTest.cs b/src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServiceMetaDataParserTest.cs index 5837853d8..bf545dcea 100644 --- a/src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServiceMetaDataParserTest.cs +++ b/src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServiceMetaDataParserTest.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Text.Json; +using NetDaemon.HassModel.CodeGenerator; using NetDaemon.HassModel.CodeGenerator.Model; namespace NetDaemon.HassModel.Tests.CodeGenerator; @@ -143,7 +144,6 @@ public void DeserializeTargetEntityArray() } - [Fact] public void NumericStepCanBeAny() { @@ -183,18 +183,30 @@ public void NumericStepCanBeAny() "name":"Longitude", "description":"Longitude of your location." }, - "elevation":{ - "required":false, - "example":120, - "selector":{ - "number":{ - "mode":"box", - "step":"1" - } - }, - "name":"Elevation", - "description":"Elevation of your location." - } + "elevation":{ + "required":false, + "example":120, + "selector":{ + "number":{ + "mode":"box", + "step":"1" + } + }, + "name":"Elevation", + "description":"Elevation of your location." + }, + "testNumberStep":{ + "required":false, + "example":120, + "selector":{ + "number":{ + "mode":"box", + "step":0.01 + } + }, + "name":"Elevation", + "description":"Elevation of your location." + } } } } @@ -203,7 +215,82 @@ public void NumericStepCanBeAny() var result = Parse(sample); var steps = result.Single().Services.Single().Fields!.Select(f => (f.Selector as NumberSelector)!.Step).ToArray(); - steps.Should().Equal(null, null, 1); // any is mapped to null + steps.Should().Equal(null, null, 1, 0.01d); // any is mapped to null + } + + + [Fact] + public void JsonError() + { + var sample = """ + { + "orbiter_services": { + "invalid_json_service": { + "name": "Observe Planet", + "description": ["Array is not allowed here!"], + "fields": { + "frequency": { + "required": 1.1, + "example": false, + "selector": { + "number": { + "multiple": false, + "mode": "box", + "min": "N/A", + "max": "N/A", + "step": "any" + } + } + } + } + }, + "navigate": { + "name": "Navigates to a new location", + "fields": { + "latitude": { + "required": true, + "example": 32.87336, + "selector": { + "number": { + "mode": 212, + "min": -90, + "max": 90, + "step": "any" + } + }, + "name": "Latitude", + "description": "Latitude of your location." + }, + "longitude": { + "required": true, + "example": 117.22743, + "selector": { + "number": { + "mode": "box", + "min": -180, + "max": 180, + "step": "any" + } + }, + "name": "Longitude", + "description": "Longitude of your location." + } + } + } + } + } + """; + var element = JsonDocument.Parse(sample).RootElement; + var result = ServiceMetaDataParser.Parse(element, out var errors); + + errors.Should().HaveCount(1, because: "We should get an error for the failed service"); + errors.Single().Context.Should().Be("orbiter_services.invalid_json_service"); + + result.Should().HaveCount(1, because:"The service that is valid should still be parsed "); + result.Single().Services.Should().HaveCount(1); + + // Just to manually validate the console output while running in the in the IDE + Controller.CheckParseErrors(errors); } private static IReadOnlyCollection Parse(string sample) diff --git a/src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServicesGeneratorTest.cs b/src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServicesGeneratorTest.cs index 5debf4013..54a1b8ad4 100644 --- a/src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServicesGeneratorTest.cs +++ b/src/HassModel/NetDaemon.HassModel.Tests/CodeGenerator/ServicesGeneratorTest.cs @@ -33,7 +33,7 @@ public void TestServicesGeneration() Service = "turn_on", Fields = new HassServiceField[] { new() { Field = "transition", Selector = new NumberSelector(), }, - new() { Field = "brightness", Selector = new NumberSelector { StepValue = "0.2" }, } + new() { Field = "brightness", Selector = new NumberSelector { Step = 0.2d }, } }, Target = new TargetSelector {