diff --git a/src/Config/Converters/Utf8JsonReaderExtensions.cs b/src/Config/Converters/Utf8JsonReaderExtensions.cs index 20c6821d02..2a1da935ae 100644 --- a/src/Config/Converters/Utf8JsonReaderExtensions.cs +++ b/src/Config/Converters/Utf8JsonReaderExtensions.cs @@ -1,11 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Text; using System.Text.Json; namespace Azure.DataApiBuilder.Config.Converters; -static internal class Utf8JsonReaderExtensions +static public class Utf8JsonReaderExtensions { /// /// Reads a string from the Utf8JsonReader by using the deserialize method rather than GetString. @@ -41,4 +42,115 @@ static internal class Utf8JsonReaderExtensions return JsonSerializer.Deserialize(ref reader, options); } + + /// + /// Replaces placeholders for environment variables in the given JSON string and returns the new JSON string. + /// + /// The JSON string to process. + /// Thrown when the input is null or when an error occurs while processing the JSON. + public static string? ReplaceEnvVarsInJson(string? inputJson) + { + try + { + if (inputJson is null) + { + throw new JsonException("Input JSON string is null."); + } + + // Load the JSON string + using JsonDocument doc = JsonDocument.Parse(inputJson); + using MemoryStream stream = new(); + using Utf8JsonWriter writer = new(stream, new JsonWriterOptions { Indented = true }); + + // Replace environment variable placeholders + ReplaceEnvVarsAndWrite(doc.RootElement, writer); + writer.Flush(); + + // return the final JSON as a string + return Encoding.UTF8.GetString(stream.ToArray()); + } + catch (Exception e) + { + throw new JsonException("Failed to replace environment variables in given JSON. " + e.Message); + } + } + + /// + /// Replaces placeholders for environment variables in a JSON element and writes the result to a JSON writer. + /// + /// The JSON element to process. + /// The JSON writer to write the result to. + private static void ReplaceEnvVarsAndWrite(JsonElement element, Utf8JsonWriter writer) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + foreach (JsonProperty property in element.EnumerateObject()) + { + writer.WritePropertyName(property.Name); + // Recursively process each property of the object + ReplaceEnvVarsAndWrite(property.Value, writer); + } + + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (JsonElement item in element.EnumerateArray()) + { + // Recursively process each item of the array + ReplaceEnvVarsAndWrite(item, writer); + } + + writer.WriteEndArray(); + break; + case JsonValueKind.String: + string? value = element.GetString(); + if (value is not null && value.StartsWith("@env('") && value.EndsWith("')")) + { + string envVar = value[6..^2]; + + // Check if the environment variable is set + if (!Environment.GetEnvironmentVariables().Contains(envVar)) + { + throw new ArgumentException($"Environment variable '{envVar}' is not set."); + } + + string? envValue = Environment.GetEnvironmentVariable(envVar); + + // Write the value of the environment variable to the JSON writer + if (envValue == "null") + { + writer.WriteNullValue(); + } + else if (bool.TryParse(envValue, out bool boolValue)) + { + writer.WriteBooleanValue(boolValue); + } + else if (int.TryParse(envValue, out int intValue)) + { + writer.WriteNumberValue(intValue); + } + else if (double.TryParse(envValue, out double doubleValue)) + { + writer.WriteNumberValue(doubleValue); + } + else + { + writer.WriteStringValue(envValue); + } + } + else + { + writer.WriteStringValue(value); + } + + break; + default: + // Write the JSON element to the writer as is + element.WriteTo(writer); + break; + } + } } diff --git a/src/Core/Configurations/RuntimeConfigValidator.cs b/src/Core/Configurations/RuntimeConfigValidator.cs index 7ffb0bd7f7..b35c08585b 100644 --- a/src/Core/Configurations/RuntimeConfigValidator.cs +++ b/src/Core/Configurations/RuntimeConfigValidator.cs @@ -3,7 +3,9 @@ using System.IO.Abstractions; using System.Net; +using System.Text.Json; using System.Text.RegularExpressions; +using Azure.DataApiBuilder.Config.Converters; using Azure.DataApiBuilder.Config.DatabasePrimitives; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Core.AuthenticationHelpers; @@ -190,7 +192,19 @@ public async Task TryValidateConfig( /// public async Task ValidateConfigSchema(RuntimeConfig runtimeConfig, string configFilePath, ILoggerFactory loggerFactory) { - string jsonData = _fileSystem.File.ReadAllText(configFilePath); + string? jsonData = _fileSystem.File.ReadAllText(configFilePath); + + // The config file may contain some environment variables that need to be replaced before validation. + try + { + jsonData = Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(jsonData); + } + catch (JsonException e) + { + _logger.LogError(e.Message); + return new JsonSchemaValidationResult(isValid: false, errors: null); + } + ILogger jsonConfigValidatorLogger = loggerFactory.CreateLogger(); JsonConfigSchemaValidator jsonConfigSchemaValidator = new(jsonConfigValidatorLogger, _fileSystem); @@ -202,7 +216,7 @@ public async Task ValidateConfigSchema(RuntimeConfig return new JsonSchemaValidationResult(isValid: false, errors: null); } - return await jsonConfigSchemaValidator.ValidateJsonConfigWithSchemaAsync(jsonSchema, jsonData); + return await jsonConfigSchemaValidator.ValidateJsonConfigWithSchemaAsync(jsonSchema, jsonData!); } /// diff --git a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs index 036b2b5450..0988968086 100644 --- a/src/Service.Tests/Unittests/SerializationDeserializationTests.cs +++ b/src/Service.Tests/Unittests/SerializationDeserializationTests.cs @@ -8,10 +8,12 @@ using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; +using Azure.DataApiBuilder.Config.Converters; using Azure.DataApiBuilder.Config.DatabasePrimitives; using Azure.DataApiBuilder.Config.ObjectModel; using Azure.DataApiBuilder.Core.Services.MetadataProviders.Converters; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; namespace Azure.DataApiBuilder.Service.Tests.Unittests { @@ -280,6 +282,120 @@ public void TestDictionaryDatabaseObjectSerializationDeserialization() VerifySourceDefinitionSerializationDeserialization(deserializedDatabaseTable.TableDefinition, _databaseTable.TableDefinition, "FirstName"); } + /// + /// This test sets environment variables of different data types, + /// creates an input JSON string with placeholders for these environment variables, + /// and then calls TryGetEnvVarsReplacedJson to replace the placeholders with the actual environment variable values. + /// The output JSON is then compared with the expected JSON to verify that the placeholders were replaced correctly. + /// + [TestMethod] + public void TestEnvVariableReplacementWithDifferentDataTypes() + { + // Set environment variables + Environment.SetEnvironmentVariable("ENV_STRING", "Hello"); + Environment.SetEnvironmentVariable("ENV_NUMBER", "123"); + Environment.SetEnvironmentVariable("ENV_FLOAT", "123.45"); + Environment.SetEnvironmentVariable("ENV_BOOLEAN", "true"); + Environment.SetEnvironmentVariable("ENV_NULL", "null"); + + // Create input JSON with environment variable placeholders + string inputJson = @" + { + ""stringKey"": ""@env('ENV_STRING')"", + ""numberKey"": ""@env('ENV_NUMBER')"", + ""floatKey"": ""@env('ENV_FLOAT')"", + ""booleanKey"": ""@env('ENV_BOOLEAN')"", + ""nullKey"": ""@env('ENV_NULL')"", + ""arrayKey"": [""@env('ENV_STRING')"", ""@env('ENV_NUMBER')"", ""@env('ENV_FLOAT')"", ""@env('ENV_BOOLEAN')"", ""@env('ENV_NULL')""], + ""objectKey"": { + ""stringKey"": ""@env('ENV_STRING')"", + ""numberKey"": ""@env('ENV_NUMBER')"", + ""floatKey"": ""@env('ENV_FLOAT')"", + ""booleanKey"": ""@env('ENV_BOOLEAN')"", + ""nullKey"": ""@env('ENV_NULL')"" + } + }"; + + // Call the method under test + string outputJson = null; + try + { + outputJson = Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(inputJson); + } + catch (JsonException e) + { + Assert.Fail("Unexpected Failure. " + e.Message); + } + + // Create expected JSON + string expectedJson = @" + { + ""stringKey"": ""Hello"", + ""numberKey"": 123, + ""floatKey"": 123.45, + ""booleanKey"": true, + ""nullKey"": null, + ""arrayKey"": [""Hello"", 123, 123.45, true, null], + ""objectKey"": { + ""stringKey"": ""Hello"", + ""numberKey"": 123, + ""floatKey"": 123.45, + ""booleanKey"": true, + ""nullKey"": null + } + }"; + + // Compare the output JSON with the expected JSON + Assert.IsTrue(JToken.DeepEquals(JToken.Parse(expectedJson), JToken.Parse(outputJson))); + } + + /// + /// This test checks that the ReplaceEnvVarsInJson method throws a JsonException when given an invalid JSON string. + /// The test asserts that the exception's message matches the expected error message. + /// + [TestMethod] + public void TestEnvVariableReplacementWithInvalidJson() + { + // Create an invalid JSON string + string inputJson = @"{ ""stringKey"": ""@env('ENV_STRING')', }"; + + // Call the method under test and expect a JsonException + try + { + Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(inputJson); + Assert.Fail("Expected a JsonException to be thrown"); + } + catch (JsonException ex) + { + Assert.AreEqual("Failed to replace environment variables in given JSON. " + + "Expected end of string, but instead reached end of data. LineNumber: 0 | BytePositionInLine: 38.", ex.Message); + } + } + + /// + /// This test validates that the `ReplaceEnvVarsInJson` method throws a JsonException when + /// it encounters a reference to an unset environment variable in the provided JSON string. + /// The test asserts that the exception's message matches the expected error message. + /// + [TestMethod] + public void TestEnvVariableReplacementWithMissingEnvVar() + { + // Create a JSON string with a placeholder for the missing environment variable + string inputJson = @"{ ""stringKey"": ""@env('MISSING_ENV_VAR')"" }"; + + // Call the method under test and expect a JsonException + try + { + Utf8JsonReaderExtensions.ReplaceEnvVarsInJson(inputJson); + Assert.Fail("Expected a JsonException to be thrown"); + } + catch (JsonException ex) + { + Assert.AreEqual("Failed to replace environment variables in given JSON. " + + "Environment variable 'MISSING_ENV_VAR' is not set.", ex.Message); + } + } + private void InitializeObjects() { _options = new()