Add MaxResponseSizeMB to runtime config.

src/Cli.Tests/ModuleInitializer.cs

@@ -55,6 +55,10 @@ public static void Init()
         VerifierSettings.IgnoreMember<DataSource>(dataSource => dataSource.DatabaseTypeNotSupportedMessage);
         // Ignore DefaultDataSourceName as that's not serialized in our config file.
         VerifierSettings.IgnoreMember<RuntimeConfig>(config => config.DefaultDataSourceName);
+        // Ignore MaxResponseSizeMB as as that's unimportant from a test standpoint.
+        VerifierSettings.IgnoreMember<HostOptions>(options => options.MaxResponseSizeMB);
+        // Ignore UserProvidedMaxResponseSizeMB as that's not serialized in our config file.
+        VerifierSettings.IgnoreMember<HostOptions>(options => options.UserProvidedMaxResponseSizeMB);
         // Customise the path where we store snapshots, so they are easier to locate in a PR review.
         VerifyBase.DerivePathInfo(
             (sourceFile, projectDirectory, type, method) => new(

src/Config/Converters/HostOptionsConverterFactory.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Azure.DataApiBuilder.Config.ObjectModel;
+
+namespace Azure.DataApiBuilder.Config.Converters;
+
+/// <summary>
+/// Defines how DAB reads and writes host options.
+/// </summary>
+internal class HostOptionsConvertorFactory : JsonConverterFactory
+{
+    /// <inheritdoc/>
+    public override bool CanConvert(Type typeToConvert)
+    {
+        return typeToConvert.IsAssignableTo(typeof(HostOptions));
+    }
+
+    /// <inheritdoc/>
+    public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
+    {
+        return new HostOptionsConverter();
+    }
+
+    private class HostOptionsConverter : JsonConverter<HostOptions>
+    {
+        /// <summary>
+        /// Defines how DAB reads host options and defines which values are
+        /// used to instantiate HostOptions.
+        /// Uses default deserialize.
+        /// </summary>
+        /// <exception cref="JsonException">Thrown when improperly formatted host options are provided.</exception>
+        public override HostOptions? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        {
+            // Remove the converter so we don't recurse.
+            JsonSerializerOptions jsonSerializerOptions = new(options);
+            jsonSerializerOptions.Converters.Remove(jsonSerializerOptions.Converters.First(c => c is HostOptionsConvertorFactory));
+            return JsonSerializer.Deserialize<HostOptions>(ref reader, jsonSerializerOptions);
+        }
+
+        /// <summary>
+        /// When writing the HostOptions back to a JSON file, only write the MaxResponseSizeMB property
+        /// if the property is user provided. This avoids polluting the written JSON file with a property
+        /// the user most likely ommitted when writing the original DAB runtime config file.
+        /// This Write operation is only used when a RuntimeConfig object is serialized to JSON.
+        /// </summary>
+        public override void Write(Utf8JsonWriter writer, HostOptions value, JsonSerializerOptions options)
+        {
+            writer.WriteStartObject();
+            writer.WritePropertyName("cors");
+            JsonSerializer.Serialize(writer, value.Cors, options);
+            writer.WritePropertyName("authentication");
+            JsonSerializer.Serialize(writer, value.Authentication, options);
+            writer.WritePropertyName("mode");
+            JsonSerializer.Serialize(writer, value.Mode, options);
+
+            if (value?.UserProvidedMaxResponseSizeMB is true)
+            {
+                writer.WritePropertyName("max-response-size-mb");
+                JsonSerializer.Serialize(writer, value.MaxResponseSizeMB, options);
+            }
+
+            writer.WriteEndObject();
+        }
+    }
+}

src/Config/ObjectModel/HostOptions.cs

@@ -1,6 +1,80 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+using System.Text.Json.Serialization;
+using Azure.DataApiBuilder.Service.Exceptions;
+
 namespace Azure.DataApiBuilder.Config.ObjectModel;
 
-public record HostOptions(CorsOptions? Cors, AuthenticationOptions? Authentication, HostMode Mode = HostMode.Production);
+public record HostOptions
+{
+    /// <summary>
+    /// Dab engine can at maximum handle 158 MB of data in a single response from a source.
+    /// Json deserialization of a response into a string has a limit of 166,666,666 bytes which when converted to MB is 158 MB.
+    /// ref: enforcing code:
+    /// .net8: https://github.com/dotnet/runtime/blob/v6.0.0/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs#L80
+    /// .net6: https://github.com/dotnet/runtime/blob/v8.0.0/src/libraries/System.Text.Json/src/System/Text/Json/Writer/JsonWriterHelper.cs#75
+    /// ref: Json constant: https://github.com/dotnet/runtime/blob/main/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs#L80
+    /// </summary>
+    private const int MAX_RESPONSE_LENGTH_DAB_ENGINE_MB = 158;
+
+    /// <summary>
+    /// Dab engine default response length. As of now this is same as max response length.
+    /// </summary>
+    private const int DEFAULT_RESPONSE_LENGTH_DAB_ENGINE_MB = 158;
+
+    [JsonPropertyName("cors")]
+    public CorsOptions? Cors { get; init; }
+
+    [JsonPropertyName("authentication")]
+    public AuthenticationOptions? Authentication { get; init; }
+
+    [JsonPropertyName("mode")]
+    public HostMode Mode { get; init; }
+
+    [JsonPropertyName("max-response-size-mb")]
+    public int? MaxResponseSizeMB { get; init; } = null;
+
+    public HostOptions(CorsOptions? Cors, AuthenticationOptions? Authentication, HostMode Mode = HostMode.Production, int? MaxResponseSizeMB = null)
+    {
+        this.Cors = Cors;
+        this.Authentication = Authentication;
+        this.Mode = Mode;
+        this.MaxResponseSizeMB = MaxResponseSizeMB;
+
+        if (this.MaxResponseSizeMB is not null)
+        {
+            this.MaxResponseSizeMB = this.MaxResponseSizeMB == -1 ? MAX_RESPONSE_LENGTH_DAB_ENGINE_MB : (int)this.MaxResponseSizeMB;
+            if (this.MaxResponseSizeMB < 1 || this.MaxResponseSizeMB > MAX_RESPONSE_LENGTH_DAB_ENGINE_MB)
+            {
+                throw new DataApiBuilderException(
+                    message: $"{nameof(RuntimeConfig.Runtime.Host.MaxResponseSizeMB)} cannot be 0, exceed {MAX_RESPONSE_LENGTH_DAB_ENGINE_MB}MB or be less than -1",
+                    statusCode: HttpStatusCode.ServiceUnavailable,
+                    subStatusCode: DataApiBuilderException.SubStatusCodes.ConfigValidationError);
+            }
+
+            UserProvidedMaxResponseSizeMB = true;
+        }
+        else
+        {
+            this.MaxResponseSizeMB = DEFAULT_RESPONSE_LENGTH_DAB_ENGINE_MB;
+        }
+    }
+
+    /// <summary>
+    /// Flag which informs CLI and JSON serializer whether to write MaxResponseSizeMB.
+    /// property and value to the runtime config file.
+    /// When user doesn't provide the MaxResponseSizeMB property/value or provides a null value, which signals DAB to use the default,
+    /// the DAB CLI will not write the default value to a serialized config.
+    /// This is because the user's intent is to use DAB's default value which could change
+    /// and DAB CLI writing the property and value would lose the user's intent.
+    /// This is because if the user were to use the CLI created config, MaxResponseSizeMB
+    /// property/value specified would be interpreted by DAB as "user explicitly set MaxResponseSizeMB.
+    /// UserProvidedMaxResponseSizeMB is true only when a user provides a non-null value
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    [MemberNotNullWhen(true, nameof(MaxResponseSizeMB))]
+    public bool UserProvidedMaxResponseSizeMB { get; init; } = false;
+}

src/Config/ObjectModel/RuntimeConfig.cs

@@ -495,6 +495,17 @@ public uint MaxPageSize()
         return (uint?)Runtime?.Pagination?.MaxPageSize ?? PaginationOptions.MAX_PAGE_SIZE;
     }
 
+    public int? MaxResponseSizeMB()
+    {
+        return Runtime?.Host?.MaxResponseSizeMB;
+    }
+
+    public bool MaxResponseSizeLogicEnabled()
+    {
+        // If the user has provided a max response size, we should use new logic to enforce it.
+        return Runtime?.Host?.UserProvidedMaxResponseSizeMB ?? false;
+    }
+
     /// <summary>
     /// Get the pagination limit from the runtime configuration.
     /// </summary>

src/Config/RuntimeConfigLoader.cs

@@ -181,6 +181,7 @@ public static JsonSerializerOptions GetSerializationOptions(
         options.Converters.Add(new MultipleCreateOptionsConverter());
         options.Converters.Add(new MultipleMutationOptionsConverter(options));
         options.Converters.Add(new DataSourceConverterFactory(replaceEnvVar));
+        options.Converters.Add(new HostOptionsConvertorFactory());
 
         if (replaceEnvVar)
         {

src/Service.Tests/ModuleInitializer.cs

@@ -59,6 +59,10 @@ public static void Init()
         VerifierSettings.IgnoreMember<DataSource>(dataSource => dataSource.DatabaseTypeNotSupportedMessage);
         // Ignore DefaultDataSourceName as that's not serialized in our config file.
         VerifierSettings.IgnoreMember<RuntimeConfig>(config => config.DefaultDataSourceName);
+        // Ignore MaxResponseSizeMB as as that's unimportant from a test standpoint.
+        VerifierSettings.IgnoreMember<HostOptions>(options => options.MaxResponseSizeMB);
+        // Ignore UserProvidedMaxResponseSizeMB as that's not serialized in our config file.
+        VerifierSettings.IgnoreMember<HostOptions>(options => options.UserProvidedMaxResponseSizeMB);
         // Customise the path where we store snapshots, so they are easier to locate in a PR review.
         VerifyBase.DerivePathInfo(
             (sourceFile, projectDirectory, type, method) => new(

src/Service.Tests/Unittests/ConfigValidationUnitTests.cs

@@ -2414,6 +2414,51 @@ public void ValidatePaginationOptionsInConfig(
             }
         }
 
+        /// <summary>
+        /// Test to validate the max response size option in the runtime config.
+        /// Note:Changing the default values of max response size would be a breaking change.
+        /// </summary>
+        /// <param name="exceptionExpected">should there be an exception.</param>
+        /// <param name="maxDbResponseSizeMB">maxResponse size input</param>
+        /// <param name="expectedExceptionMessage">expected exception message in case there is exception.</param>
+        /// <param name="expectedMaxResponseSize">expected value in config.</param>
+        [DataTestMethod]
+        [DataRow(null, 158, false, "",
+            DisplayName = $"{nameof(RuntimeConfig.Runtime.Host.MaxResponseSizeMB)} should be 158MB when no value provided in config.")]
+        [DataRow(64, 64, false, "",
+            DisplayName = $"Valid positive input of {nameof(RuntimeConfig.Runtime.Host.MaxResponseSizeMB)}  > 0 and <= 158MB must be accepted and set in the config.")]
+        [DataRow(-1, 158, false, "",
+            DisplayName = $"-1 user input for {nameof(RuntimeConfig.Runtime.Host.MaxResponseSizeMB)} should result in a value of 158MB which is the max value supported by dab engine")]
+        [DataRow(0, null, true, $"{nameof(RuntimeConfig.Runtime.Host.MaxResponseSizeMB)} cannot be 0, exceed 158MB or be less than -1",
+            DisplayName = $"Input of 0 for {nameof(RuntimeConfig.Runtime.Host.MaxResponseSizeMB)} must throw exception.")]
+        [DataRow(159, null, true, $"{nameof(RuntimeConfig.Runtime.Host.MaxResponseSizeMB)} cannot be 0, exceed 158MB or be less than -1",
+            DisplayName = $"Inputs of {nameof(RuntimeConfig.Runtime.Host.MaxResponseSizeMB)} greater than 158MB must throw exception.")]
+        public void ValidateMaxResponseSizeInConfig(
+            int? providedMaxResponseSizeMB,
+            int? expectedMaxResponseSizeMB,
+            bool isExceptionExpected,
+            string expectedExceptionMessage)
+        {
+            try
+            {
+                RuntimeConfig runtimeConfig = new(
+                    Schema: "UnitTestSchema",
+                    DataSource: new DataSource(DatabaseType: DatabaseType.MSSQL, "", Options: null),
+                    Runtime: new(
+                        Rest: new(),
+                        GraphQL: new(),
+                        Host: new(Cors: null, Authentication: null, MaxResponseSizeMB: providedMaxResponseSizeMB)
+                    ),
+                    Entities: new(new Dictionary<string, Entity>()));
+                Assert.AreEqual(expectedMaxResponseSizeMB, runtimeConfig.MaxResponseSizeMB());
+            }
+            catch (DataApiBuilderException ex)
+            {
+                Assert.IsTrue(isExceptionExpected);
+                Assert.AreEqual(expectedExceptionMessage, ex.Message);
+            }
+        }
+
         private static RuntimeConfigValidator InitializeRuntimeConfigValidator()
         {
             MockFileSystem fileSystem = new();