Adding pagination limits.

src/Config/ObjectModel/PaginationOptions.cs

@@ -0,0 +1,112 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+using System.Text.Json.Serialization;
+using Azure.DataApiBuilder.Service.Exceptions;
+
+namespace Azure.DataApiBuilder.Config.ObjectModel;
+
+/// <summary>
+/// Pagination options for the dab setup.
+/// Properties are nullable to support DAB CLI merge config
+/// expected behavior.
+/// </summary>
+public record PaginationOptions
+{
+    /// <summary>
+    /// Default page size.
+    /// </summary>
+    public const uint DEFAULT_PAGE_SIZE = 100;
+
+    /// <summary>
+    /// Max page size.
+    /// </summary>
+    public const uint MAX_PAGE_SIZE = 100000;
+
+    /// <summary>
+    /// The default page size for pagination.
+    /// </summary>
+    [JsonPropertyName("default-page-size")]
+    public int? DefaultPageSize { get; init; } = null;
+
+    /// <summary>
+    /// The max page size for pagination.
+    /// </summary>
+    [JsonPropertyName("max-page-size")]
+    public int? MaxPageSize { get; init; } = null;
+
+    [JsonConstructor]
+    public PaginationOptions(int? DefaultPageSize = null, int? MaxPageSize = null)
+    {
+        if (MaxPageSize is not null)
+        {
+            ValidatePageSize((int)MaxPageSize);
+            this.MaxPageSize = MaxPageSize == -1 ? Int32.MaxValue : (int)MaxPageSize;
+            UserProvidedMaxPageSize = true;
+        }
+        else
+        {
+            this.MaxPageSize = (int)MAX_PAGE_SIZE;
+        }
+
+        if (DefaultPageSize is not null)
+        {
+            ValidatePageSize((int)DefaultPageSize);
+            this.DefaultPageSize = DefaultPageSize == -1 ? (int)this.MaxPageSize : (int)DefaultPageSize;
+            UserProvidedDefaultPageSize = true;
+        }
+        else
+        {
+            this.DefaultPageSize = (int)DEFAULT_PAGE_SIZE;
+        }
+
+        if (this.DefaultPageSize > this.MaxPageSize)
+        {
+            throw new DataApiBuilderException(
+                message: "Pagination options invalid. The default page size cannot be greater than max page size",
+                statusCode: HttpStatusCode.ServiceUnavailable,
+                subStatusCode: DataApiBuilderException.SubStatusCodes.ConfigValidationError);
+        }
+    }
+
+    /// <summary>
+    /// Flag which informs CLI and JSON serializer whether to write default page size.
+    /// property and value to the runtime config file.
+    /// When user doesn't provide the default-page-size property/value, which signals DAB to use the default,
+    /// the DAB CLI should not write the default value to a serialized config.
+    /// This is because the user's intent is to use DAB's default value which could change
+    /// and DAB CLI writing the property and value would lose the user's intent.
+    /// This is because if the user were to use the CLI created config, a default-page-size
+    /// property/value specified would be interpreted by DAB as "user explicitly default-page-size."
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    [MemberNotNullWhen(true, nameof(DefaultPageSize))]
+    public bool UserProvidedDefaultPageSize { get; init; } = false;
+
+    /// <summary>
+    /// Flag which informs CLI and JSON serializer whether to write max-page-size
+    /// property and value to the runtime config file.
+    /// When user doesn't provide the max-page-size property/value, which signals DAB to use the default,
+    /// the DAB CLI should not write the default value to a serialized config.
+    /// This is because the user's intent is to use DAB's default value which could change
+    /// and DAB CLI writing the property and value would lose the user's intent.
+    /// This is because if the user were to use the CLI created config, a max-page-size
+    /// property/value specified would be interpreted by DAB as "user explicitly max-page-size."
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    [MemberNotNullWhen(true, nameof(MaxPageSize))]
+    public bool UserProvidedMaxPageSize { get; init; } = false;
+
+    private static void ValidatePageSize(int pageSize)
+    {
+        if (pageSize < -1 || pageSize == 0 || pageSize > Int32.MaxValue)
+        {
+            throw new DataApiBuilderException(
+                message: "Pagination options invalid. Page size arguments cannot be 0, exceed max int value or be less than -1",
+                statusCode: HttpStatusCode.ServiceUnavailable,
+                subStatusCode: DataApiBuilderException.SubStatusCodes.ConfigValidationError);
+        }
+    }
+}

src/Config/ObjectModel/RuntimeConfig.cs

@@ -471,4 +471,45 @@ Runtime.GraphQL.MultipleMutationOptions is not null &&
                Runtime.GraphQL.MultipleMutationOptions.MultipleCreateOptions is not null &&
                Runtime.GraphQL.MultipleMutationOptions.MultipleCreateOptions.Enabled);
     }
+
+    public uint DefaultPageSize()
+    {
+        return (uint?)Runtime?.Pagination?.DefaultPageSize ?? PaginationOptions.DEFAULT_PAGE_SIZE;
+    }
+
+    public uint MaxPageSize()
+    {
+        return (uint?)Runtime?.Pagination?.MaxPageSize ?? PaginationOptions.MAX_PAGE_SIZE;
+    }
+
+    /// <summary>
+    /// Get the pagination limit from the runtime configuration.
+    /// </summary>
+    /// <param name="first">The pagination input from the user. Example: $first=10</param>
+    /// <returns></returns>
+    /// <exception cref="DataApiBuilderException"></exception>
+    public uint GetPaginationLimit(int? first)
+    {
+        uint defaultPageSize = this.DefaultPageSize();
+        uint maxPageSize = this.MaxPageSize();
+
+        if (first is not null)
+        {
+            if (first < -1 || first == 0 || first > maxPageSize)
+            {
+                throw new DataApiBuilderException(
+                message: $"Invalid number of items requested, {nameof(first)} argument must be either -1 or a positive number within the max page size limit of {maxPageSize}. Actual value: {first}",
+                statusCode: HttpStatusCode.BadRequest,
+                subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest);
+            }
+            else
+            {
+                return (first == -1 ? maxPageSize : (uint)first);
+            }
+        }
+        else
+        {
+            return defaultPageSize;
+        }
+    }
 }

src/Config/ObjectModel/RuntimeOptions.cs

@@ -14,6 +14,7 @@ public record RuntimeOptions
     public string? BaseRoute { get; init; }
     public TelemetryOptions? Telemetry { get; init; }
     public EntityCacheOptions? Cache { get; init; }
+    public PaginationOptions? Pagination { get; init; }
 
     [JsonConstructor]
     public RuntimeOptions(
@@ -22,14 +23,16 @@ public RuntimeOptions(
         HostOptions? Host,
         string? BaseRoute = null,
         TelemetryOptions? Telemetry = null,
-        EntityCacheOptions? Cache = null)
+        EntityCacheOptions? Cache = null,
+        PaginationOptions? Pagination = null)
     {
         this.Rest = Rest;
         this.GraphQL = GraphQL;
         this.Host = Host;
         this.BaseRoute = BaseRoute;
         this.Telemetry = Telemetry;
         this.Cache = Cache;
+        this.Pagination = Pagination;
     }
 
     /// <summary>

src/Core/Models/GraphQLFilterParsers.cs

@@ -315,6 +315,7 @@ private void HandleNestedFilterForCosmos(
         CosmosExistsQueryStructure existsQuery = new(
             ctx,
             new Dictionary<string, object?>(),
+            _configProvider,
             metadataProvider,
             queryStructure.AuthorizationResolver,
             this,

src/Core/Models/RestRequestContexts/RestRequestContext.cs

@@ -88,7 +88,7 @@ protected RestRequestContext(string entityName, DatabaseObject dbo)
     /// Based on request this property may or may not be populated.
     /// </summary>
 
-    public uint? First { get; set; }
+    public int? First { get; set; }
     /// <summary>
     /// Is the result supposed to be multiple values or not.
     /// </summary>

src/Core/Resolvers/CosmosExistsQueryStructure.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using Azure.DataApiBuilder.Auth;
+using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Services;
 using HotChocolate.Resolvers;
@@ -15,13 +16,15 @@ public class CosmosExistsQueryStructure : CosmosQueryStructure
         /// </summary>
         public CosmosExistsQueryStructure(IMiddlewareContext context,
             IDictionary<string, object?> parameters,
+            RuntimeConfigProvider runtimeConfigProvider,
             ISqlMetadataProvider metadataProvider,
             IAuthorizationResolver authorizationResolver,
             GQLFilterParser gQLFilterParser,
             IncrementingInteger? counter = null,
             List<Predicate>? predicates = null)
             : base(context,
                   parameters,
+                  runtimeConfigProvider,
                   metadataProvider,
                   authorizationResolver,
                   gQLFilterParser,

src/Core/Resolvers/CosmosQueryEngine.cs

@@ -69,7 +69,7 @@ public async Task<Tuple<JsonDocument, IMetadata>> ExecuteAsync(
 
             ISqlMetadataProvider metadataStoreProvider = _metadataProviderFactory.GetMetadataProvider(dataSourceName);
 
-            CosmosQueryStructure structure = new(context, parameters, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
+            CosmosQueryStructure structure = new(context, parameters, _runtimeConfigProvider, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
             RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig();
 
             string queryString = _queryBuilder.Build(structure);
@@ -201,7 +201,7 @@ public async Task<Tuple<IEnumerable<JsonDocument>, IMetadata>> ExecuteListAsync(
             // TODO: add support for TOP and Order-by push-down
 
             ISqlMetadataProvider metadataStoreProvider = _metadataProviderFactory.GetMetadataProvider(dataSourceName);
-            CosmosQueryStructure structure = new(context, parameters, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
+            CosmosQueryStructure structure = new(context, parameters, _runtimeConfigProvider, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
             CosmosClient client = _clientProvider.Clients[dataSourceName];
             Container container = client.GetDatabase(structure.Database).GetContainer(structure.Container);
             QueryDefinition querySpec = new(_queryBuilder.Build(structure));

src/Core/Resolvers/CosmosQueryStructure.cs

@@ -5,6 +5,7 @@
 using Azure.DataApiBuilder.Auth;
 using Azure.DataApiBuilder.Config.DatabasePrimitives;
 using Azure.DataApiBuilder.Config.ObjectModel;
+using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Services;
 using Azure.DataApiBuilder.Core.Services.MetadataProviders;
@@ -36,10 +37,12 @@ public class CosmosQueryStructure : BaseQueryStructure
         public string Container { get; internal set; }
         public string Database { get; internal set; }
         public string? Continuation { get; internal set; }
-        public int? MaxItemCount { get; internal set; }
+        public uint? MaxItemCount { get; internal set; }
         public string? PartitionKeyValue { get; internal set; }
         public List<OrderByColumn> OrderByColumns { get; internal set; }
 
+        public RuntimeConfigProvider RuntimeConfigProvider { get; internal set; }
+
         public string GetTableAlias()
         {
             return $"table{TableCounter.Next()}";
@@ -48,6 +51,7 @@ public string GetTableAlias()
         public CosmosQueryStructure(
             IMiddlewareContext context,
             IDictionary<string, object?> parameters,
+            RuntimeConfigProvider provider,
             ISqlMetadataProvider metadataProvider,
             IAuthorizationResolver authorizationResolver,
             GQLFilterParser gQLFilterParser,
@@ -58,6 +62,7 @@ public CosmosQueryStructure(
             _context = context;
             SourceAlias = _containerAlias;
             DatabaseObject.Name = _containerAlias;
+            RuntimeConfigProvider = provider;
             Init(parameters);
         }
 
@@ -116,7 +121,6 @@ private void Init(IDictionary<string, object?> queryParams)
 
             IsPaginated = QueryBuilder.IsPaginationType(underlyingType);
             OrderByColumns = new();
-
             if (IsPaginated)
             {
                 FieldNode? fieldNode = ExtractItemsQueryField(selection.SyntaxNode);
@@ -155,13 +159,21 @@ private void Init(IDictionary<string, object?> queryParams)
                     (CosmosSqlMetadataProvider)MetadataProvider);
             }
 
+            RuntimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
             // first and after will not be part of query parameters. They will be going into headers instead.
             // TODO: Revisit 'first' while adding support for TOP queries
             if (queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
             {
-                MaxItemCount = (int?)queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
+                object? firstArgument = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
+                MaxItemCount = runtimeConfig?.GetPaginationLimit((int?)firstArgument);
+
                 queryParams.Remove(QueryBuilder.PAGE_START_ARGUMENT_NAME);
             }
+            else
+            {
+                // set max item count to default value.
+                MaxItemCount = runtimeConfig?.DefaultPageSize();
+            }
 
             if (queryParams.ContainsKey(QueryBuilder.PAGINATION_TOKEN_ARGUMENT_NAME))
             {

src/Core/Resolvers/Sql Query Structures/SqlQueryStructure.cs

@@ -17,7 +17,6 @@
 using HotChocolate.Language;
 using HotChocolate.Resolvers;
 using Microsoft.AspNetCore.Http;
-
 namespace Azure.DataApiBuilder.Core.Resolvers
 {
     /// <summary>
@@ -59,15 +58,10 @@ public class SqlQueryStructure : BaseSqlQueryStructure
         /// </summary>
         public Dictionary<string, string> ColumnLabelToParam { get; }
 
-        /// <summary>
-        /// Default limit when no first param is specified for list queries
-        /// </summary>
-        private const uint DEFAULT_LIST_LIMIT = 100;
-
         /// <summary>
         /// The maximum number of results this query should return.
         /// </summary>
-        private uint? _limit = DEFAULT_LIST_LIMIT;
+        private uint? _limit = PaginationOptions.DEFAULT_PAGE_SIZE;
 
         /// <summary>
         /// If this query is built because of a GraphQL query (as opposed to
@@ -198,7 +192,9 @@ public SqlQueryStructure(
             }
 
             AddColumnsForEndCursor();
-            _limit = context.First is not null ? context.First + 1 : DEFAULT_LIST_LIMIT + 1;
+            runtimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
+            _limit = runtimeConfig?.GetPaginationLimit((int?)context.First) + 1;
+
             ParametrizeColumns();
         }
 
@@ -334,24 +330,19 @@ private SqlQueryStructure(
                 IsListQuery = outputType.IsListType();
             }
 
-            if (IsListQuery && queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
+            if (IsListQuery)
             {
-                // parse first parameter for all list queries
-                object? firstObject = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
-
-                if (firstObject != null)
+                runtimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
+                if (queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
                 {
-                    int first = (int)firstObject;
-
-                    if (first <= 0)
-                    {
-                        throw new DataApiBuilderException(
-                        message: $"Invalid number of items requested, {QueryBuilder.PAGE_START_ARGUMENT_NAME} argument must be an integer greater than 0 for {schemaField.Name}. Actual value: {first.ToString()}",
-                        statusCode: HttpStatusCode.BadRequest,
-                        subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest);
-                    }
-
-                    _limit = (uint)first;
+                    // parse first parameter for all list queries
+                    object? firstObject = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
+                    _limit = runtimeConfig?.GetPaginationLimit((int?)firstObject);
+                }
+                else
+                {
+                    // if first is not passed, we should use the default page size.
+                    _limit = runtimeConfig?.DefaultPageSize();
                 }
             }