From ec45d820f1b3253cb4b36275842adff12a00efa5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 2 May 2024 23:16:37 +0530 Subject: [PATCH 1/4] fixing issue with json schema validation with env variables --- .../Converters/Utf8JsonReaderExtensions.cs | 109 ++++++++++++++++- .../Configurations/RuntimeConfigValidator.cs | 17 ++- .../SerializationDeserializationTests.cs | 114 ++++++++++++++++++ 3 files changed, 237 insertions(+), 3 deletions(-) diff --git a/src/Config/Converters/Utf8JsonReaderExtensions.cs b/src/Config/Converters/Utf8JsonReaderExtensions.cs index 20c6821d02..5b8ce801ae 100644 --- a/src/Config/Converters/Utf8JsonReaderExtensions.cs +++ b/src/Config/Converters/Utf8JsonReaderExtensions.cs @@ -1,11 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Text; using System.Text.Json; namespace Azure.DataApiBuilder.Config.Converters; -static internal class Utf8JsonReaderExtensions +static public class Utf8JsonReaderExtensions { /// /// Reads a string from the Utf8JsonReader by using the deserialize method rather than GetString. @@ -41,4 +42,110 @@ static internal class Utf8JsonReaderExtensions return JsonSerializer.Deserialize(ref reader, options); } + + /// + /// Replaces placeholders for environment variables in the given JSON string and returns the new JSON string. + /// + /// The JSON string to process. + /// Thrown when the input is null or when an error occurs while processing the JSON. + public static string? ReplaceEnvVarsInJson(string? inputJson){ + try + { + if (inputJson is null) + { + throw new JsonException("Input JSON string is null."); + } + + // Load the JSON string + using JsonDocument doc = JsonDocument.Parse(inputJson); + using MemoryStream stream = new(); + using Utf8JsonWriter writer = new(stream, new JsonWriterOptions { Indented = true }); + + // Replace environment variable placeholders + ReplaceEnvVarsAndWrite(doc.RootElement, writer); + writer.Flush(); + + // return the final JSON as a string + return Encoding.UTF8.GetString(stream.ToArray()); + } + catch (Exception e) + { + throw new JsonException("Failed to replace environment variables in given JSON. " + e.Message); + } + } + + /// + /// Replaces placeholders for environment variables in a JSON element and writes the result to a JSON writer. + /// + /// The JSON element to process. + /// The JSON writer to write the result to. + private static void ReplaceEnvVarsAndWrite(JsonElement element, Utf8JsonWriter writer) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + foreach (JsonProperty property in element.EnumerateObject()) + { + writer.WritePropertyName(property.Name); + // Recursively process each property of the object + ReplaceEnvVarsAndWrite(property.Value, writer); + } + + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (JsonElement item in element.EnumerateArray()) + { + // Recursively process each item of the array + ReplaceEnvVarsAndWrite(item, writer); + } + + writer.WriteEndArray(); + break; + case JsonValueKind.String: + string? value = element.GetString(); + if (value is not null && value.StartsWith("@env('") && value.EndsWith("')")) + { + string envVar = value[6..^2]; + + // Check if the environment variable is set + if (!Environment.GetEnvironmentVariables().Contains(envVar)) + { + throw new ArgumentException($"Environment variable '{envVar}' is not set."); + } + + string? envValue = Environment.GetEnvironmentVariable(envVar); + + // Write the value of the environment variable to the JSON writer + if (bool.TryParse(envValue, out bool boolValue)) + { + writer.WriteBooleanValue(boolValue); + } + else if (int.TryParse(envValue, out int intValue)) + { + writer.WriteNumberValue(intValue); + } + else if (double.TryParse(envValue, out double doubleValue)) + { + writer.WriteNumberValue(doubleValue); + } + else + { + writer.WriteStringValue(envValue); + } + } + else + { + writer.WriteStringValue(value); + } + + break; + default: + // Write the JSON element to the writer as is + element.WriteTo(writer); + break; + } + } } diff --git a/src/Core/Configurations/RuntimeConfigValidator.cs b/src/Core/Configurations/RuntimeConfigValidator.cs index 17ca831312..24ef29437f 100644 --- a/src/Core/Configurations/RuntimeConfigValidator.cs +++ b/src/Core/Configurations/RuntimeConfigValidator.cs @@ -16,6 +16,8 @@ using Azure.DataApiBuilder.Service.Exceptions; using Azure.DataApiBuilder.Service.GraphQLBuilder; using Microsoft.Extensions.Logging; +using Azure.DataApiBuilder.Config.Converters; +using System.Text.Json; namespace Azure.DataApiBuilder.Core.Configurations; @@ -193,7 +195,18 @@ public async Task TryValidateConfig( /// public async Task ValidateConfigSchema(RuntimeConfig runtimeConfig, string configFilePath, ILoggerFactory loggerFactory) { - string jsonData = _fileSystem.File.ReadAllText(configFilePath); + string? jsonData = _fileSystem.File.ReadAllText(configFilePath); + + // The config file may contain some environment variables that need to be replaced before validation. + try{ + jsonData = Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(jsonData); + } + catch (JsonException e) + { + _logger.LogError(e.Message); + return new JsonSchemaValidationResult(isValid: false, errors: null); + } + ILogger jsonConfigValidatorLogger = loggerFactory.CreateLogger(); JsonConfigSchemaValidator jsonConfigSchemaValidator = new(jsonConfigValidatorLogger, _fileSystem); @@ -205,7 +218,7 @@ public async Task ValidateConfigSchema(RuntimeConfig return new JsonSchemaValidationResult(isValid: false, errors: null); } - return await jsonConfigSchemaValidator.ValidateJsonConfigWithSchemaAsync(jsonSchema, jsonData); + return await jsonConfigSchemaValidator.ValidateJsonConfigWithSchemaAsync(jsonSchema, jsonData!); } /// diff --git a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs index e0b125dff7..f836c1a567 100644 --- a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs @@ -7,10 +7,12 @@ using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; +using Azure.DataApiBuilder.Config.Converters; using Azure.DataApiBuilder.Config.DatabasePrimitives; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Core.Services.MetadataProviders.Converters; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; namespace Azure.DataApiBuilder.Service.Tests.Unittests { @@ -276,6 +278,118 @@ public void TestDictionaryDatabaseObjectSerializationDeserialization() VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "FirstName"); } + /// + /// This test sets environment variables of different data types, + /// creates an input JSON string with placeholders for these environment variables, + /// and then calls TryGetEnvVarsReplacedJson to replace the placeholders with the actual environment variable values. + /// The output JSON is then compared with the expected JSON to verify that the placeholders were replaced correctly. + /// + [TestMethod] + public void TestEnvVariableReplacementWithDifferentDataTypes() + { + // Set environment variables + Environment.SetEnvironmentVariable("ENV_STRING", "Hello"); + Environment.SetEnvironmentVariable("ENV_NUMBER", "123"); + Environment.SetEnvironmentVariable("ENV_FLOAT", "123.45"); + Environment.SetEnvironmentVariable("ENV_BOOLEAN", "true"); + Environment.SetEnvironmentVariable("ENV_NULL", null); + + // Create input JSON with environment variable placeholders + string inputJson = @" + { + ""stringKey"": ""@env('ENV_STRING')"", + ""numberKey"": ""@env('ENV_NUMBER')"", + ""floatKey"": ""@env('ENV_FLOAT')"", + ""booleanKey"": ""@env('ENV_BOOLEAN')"", + ""nullKey"": ""@env('ENV_NULL')"", + ""arrayKey"": [""@env('ENV_STRING')"", ""@env('ENV_NUMBER')"", ""@env('ENV_FLOAT')"", ""@env('ENV_BOOLEAN')"", ""@env('ENV_NULL')""], + ""objectKey"": { + ""stringKey"": ""@env('ENV_STRING')"", + ""numberKey"": ""@env('ENV_NUMBER')"", + ""floatKey"": ""@env('ENV_FLOAT')"", + ""booleanKey"": ""@env('ENV_BOOLEAN')"", + ""nullKey"": ""@env('ENV_NULL')"" + } + }"; + + // Call the method under test + string outputJson=null; + try{ + outputJson = Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(inputJson); + } + catch(JsonException e){ + Assert.Fail("Unexpected Failure. " + e.Message); + } + + // Create expected JSON + string expectedJson = @" + { + ""stringKey"": ""Hello"", + ""numberKey"": 123, + ""floatKey"": 123.45, + ""booleanKey"": true, + ""nullKey"": null, + ""arrayKey"": [""Hello"", 123, 123.45, true, null], + ""objectKey"": { + ""stringKey"": ""Hello"", + ""numberKey"": 123, + ""floatKey"": 123.45, + ""booleanKey"": true, + ""nullKey"": null + } + }"; + + // Compare the output JSON with the expected JSON + Assert.IsTrue(JToken.DeepEquals(JToken.Parse(expectedJson), JToken.Parse(outputJson))); + } + + /// + /// This test checks that the ReplaceEnvVarsInJson method throws a JsonException when given an invalid JSON string. + /// The test asserts that the exception's message matches the expected error message. + /// + [TestMethod] + public void TestEnvVariableReplacementWithInvalidJson() + { + // Create an invalid JSON string + string inputJson = @"{ ""stringKey"": ""@env('ENV_STRING')', }"; + + // Call the method under test and expect a JsonException + try + { + Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(inputJson); + Assert.Fail("Expected a JsonException to be thrown"); + } + catch (JsonException ex) + { + Assert.AreEqual("Failed to replace environment variables in given JSON. " + +"Expected end of string, but instead reached end of data. LineNumber: 0 | BytePositionInLine: 38.", ex.Message); + } + } + + /// + /// This test validates that the `ReplaceEnvVarsInJson` method throws a JsonException when + /// it encounters a placeholder for an unset environment variable in the provided JSON string. + /// The test asserts that the exception's message matches the expected error message. + /// + [TestMethod] + public void TestEnvVariableReplacementWithMissingEnvVar() + { + // Create a JSON string with a placeholder for the missing environment variable + string inputJson = @"{ ""stringKey"": ""@env('MISSING_ENV_VAR')"" }"; + + // Call the method under test and expect a JsonException + try + { + Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(inputJson); + Assert.Fail("Expected a JsonException to be thrown"); + } + catch (JsonException ex) + { + Assert.AreEqual("Failed to replace environment variables in given JSON. " + +"Environment variable 'MISSING_ENV_VAR' is not set.", ex.Message); + } + } + private void InitializeObjects() { _options = new() From bb767ab95881399aa16503ef03c1c790da78b677 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 2 May 2024 23:29:52 +0530 Subject: [PATCH 2/4] fix formatting --- src/Config/Converters/Utf8JsonReaderExtensions.cs | 3 ++- src/Core/Configurations/RuntimeConfigValidator.cs | 7 ++++--- .../Unittests/SerializationDeserializationTests.cs | 12 +++++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Config/Converters/Utf8JsonReaderExtensions.cs b/src/Config/Converters/Utf8JsonReaderExtensions.cs index 5b8ce801ae..0bbfd8168c 100644 --- a/src/Config/Converters/Utf8JsonReaderExtensions.cs +++ b/src/Config/Converters/Utf8JsonReaderExtensions.cs @@ -48,7 +48,8 @@ static public class Utf8JsonReaderExtensions /// /// The JSON string to process. /// Thrown when the input is null or when an error occurs while processing the JSON. - public static string? ReplaceEnvVarsInJson(string? inputJson){ + public static string? ReplaceEnvVarsInJson(string? inputJson) + { try { if (inputJson is null) diff --git a/src/Core/Configurations/RuntimeConfigValidator.cs b/src/Core/Configurations/RuntimeConfigValidator.cs index 24ef29437f..f9aedbef6a 100644 --- a/src/Core/Configurations/RuntimeConfigValidator.cs +++ b/src/Core/Configurations/RuntimeConfigValidator.cs @@ -3,7 +3,9 @@ using System.IO.Abstractions; using System.Net; +using System.Text.Json; using System.Text.RegularExpressions; +using Azure.DataApiBuilder.Config.Converters; using Azure.DataApiBuilder.Config.DatabasePrimitives; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Core.AuthenticationHelpers; @@ -16,8 +18,6 @@ using Azure.DataApiBuilder.Service.Exceptions; using Azure.DataApiBuilder.Service.GraphQLBuilder; using Microsoft.Extensions.Logging; -using Azure.DataApiBuilder.Config.Converters; -using System.Text.Json; namespace Azure.DataApiBuilder.Core.Configurations; @@ -198,7 +198,8 @@ public async Task ValidateConfigSchema(RuntimeConfig string? jsonData = _fileSystem.File.ReadAllText(configFilePath); // The config file may contain some environment variables that need to be replaced before validation. - try{ + try + { jsonData = Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(jsonData); } catch (JsonException e) diff --git a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs index f836c1a567..84822b4e91 100644 --- a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs @@ -313,11 +313,13 @@ public void TestEnvVariableReplacementWithDifferentDataTypes() }"; // Call the method under test - string outputJson=null; - try{ + string outputJson = null; + try + { outputJson = Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(inputJson); } - catch(JsonException e){ + catch (JsonException e) + { Assert.Fail("Unexpected Failure. " + e.Message); } @@ -362,7 +364,7 @@ public void TestEnvVariableReplacementWithInvalidJson() catch (JsonException ex) { Assert.AreEqual("Failed to replace environment variables in given JSON. " - +"Expected end of string, but instead reached end of data. LineNumber: 0 | BytePositionInLine: 38.", ex.Message); + + "Expected end of string, but instead reached end of data. LineNumber: 0 | BytePositionInLine: 38.", ex.Message); } } @@ -386,7 +388,7 @@ public void TestEnvVariableReplacementWithMissingEnvVar() catch (JsonException ex) { Assert.AreEqual("Failed to replace environment variables in given JSON. " - +"Environment variable 'MISSING_ENV_VAR' is not set.", ex.Message); + + "Environment variable 'MISSING_ENV_VAR' is not set.", ex.Message); } } From e02177a515c1ffdea95d271d57c8c76757acef2e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 3 May 2024 00:12:11 +0530 Subject: [PATCH 3/4] fixing issue with null values --- src/Config/Converters/Utf8JsonReaderExtensions.cs | 6 +++++- .../Unittests/SerializationDeserializationTests.cs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Config/Converters/Utf8JsonReaderExtensions.cs b/src/Config/Converters/Utf8JsonReaderExtensions.cs index 0bbfd8168c..2a1da935ae 100644 --- a/src/Config/Converters/Utf8JsonReaderExtensions.cs +++ b/src/Config/Converters/Utf8JsonReaderExtensions.cs @@ -120,7 +120,11 @@ private static void ReplaceEnvVarsAndWrite(JsonElement element, Utf8JsonWriter w string? envValue = Environment.GetEnvironmentVariable(envVar); // Write the value of the environment variable to the JSON writer - if (bool.TryParse(envValue, out bool boolValue)) + if (envValue == "null") + { + writer.WriteNullValue(); + } + else if (bool.TryParse(envValue, out bool boolValue)) { writer.WriteBooleanValue(boolValue); } diff --git a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs index 84822b4e91..bab87f7824 100644 --- a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs @@ -292,7 +292,7 @@ public void TestEnvVariableReplacementWithDifferentDataTypes() Environment.SetEnvironmentVariable("ENV_NUMBER", "123"); Environment.SetEnvironmentVariable("ENV_FLOAT", "123.45"); Environment.SetEnvironmentVariable("ENV_BOOLEAN", "true"); - Environment.SetEnvironmentVariable("ENV_NULL", null); + Environment.SetEnvironmentVariable("ENV_NULL", "null"); // Create input JSON with environment variable placeholders string inputJson = @" From 850abd7b64a2a755bc9c3a5a2306ca58a8a7d389 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar <102276754+abhishekkumams@users.noreply.github.com> Date: Thu, 9 May 2024 10:30:00 +0530 Subject: [PATCH 4/4] Update src/Service.Tests/Unittests/SerializationDeserializationTests.cs Co-authored-by: Sean Leonard --- .../Unittests/SerializationDeserializationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs index bab87f7824..04af953723 100644 --- a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs @@ -370,7 +370,7 @@ public void TestEnvVariableReplacementWithInvalidJson() /// /// This test validates that the `ReplaceEnvVarsInJson` method throws a JsonException when - /// it encounters a placeholder for an unset environment variable in the provided JSON string. + /// it encounters a reference to an unset environment variable in the provided JSON string. /// The test asserts that the exception's message matches the expected error message. /// [TestMethod]