From daf9ce04ea8b4a00cc42e9e3b604ae897e686849 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Wed, 27 Jul 2022 15:55:32 +0530 Subject: [PATCH 01/77] Add trigger binding support (Jul 15) --- README.md | 119 ++- .../TriggerBindingSamples/ProductsTrigger.cs | 26 + ...oductsWithMultiplePrimaryColumnsTrigger.cs | 35 + src/SqlBindingConfigProvider.cs | 8 +- src/TriggerBinding/SqlChange.cs | 49 ++ src/TriggerBinding/SqlTableChangeMonitor.cs | 774 ++++++++++++++++++ src/TriggerBinding/SqlTriggerAttribute.cs | 42 + .../SqlTriggerAttributeBindingProvider.cs | 110 +++ src/TriggerBinding/SqlTriggerBinding.cs | 196 +++++ src/TriggerBinding/SqlTriggerConstants.cs | 14 + src/TriggerBinding/SqlTriggerListener.cs | 364 ++++++++ .../SqlTriggerParameterDescriptor.cs | 28 + test/Integration/IntegrationTestBase.cs | 48 +- .../SqlTriggerBindingIntegrationTests.cs | 175 ++++ test/Unit/SqlInputBindingTests.cs | 9 +- test/Unit/SqlTriggerBindingTests.cs | 106 +++ 16 files changed, 2081 insertions(+), 22 deletions(-) create mode 100644 samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs create mode 100644 samples/samples-csharp/TriggerBindingSamples/ProductsWithMultiplePrimaryColumnsTrigger.cs create mode 100644 src/TriggerBinding/SqlChange.cs create mode 100644 src/TriggerBinding/SqlTableChangeMonitor.cs create mode 100644 src/TriggerBinding/SqlTriggerAttribute.cs create mode 100644 src/TriggerBinding/SqlTriggerAttributeBindingProvider.cs create mode 100644 src/TriggerBinding/SqlTriggerBinding.cs create mode 100644 src/TriggerBinding/SqlTriggerConstants.cs create mode 100644 src/TriggerBinding/SqlTriggerListener.cs create mode 100644 src/TriggerBinding/SqlTriggerParameterDescriptor.cs create mode 100644 test/Integration/SqlTriggerBindingIntegrationTests.cs create mode 100644 test/Unit/SqlTriggerBindingTests.cs diff --git a/README.md b/README.md index 184d95659..b5cf92db6 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ This repository contains the Azure SQL binding for Azure Functions extension cod - **Input Binding**: takes a SQL query to run and returns the output of the query in the function. - **Output Binding**: takes a list of rows and upserts them into the user table (i.e. If a row doesn't already exist, it is added. If it does, it is updated). +- **Trigger Binding**: monitors the user table for changes (i.e., row inserts, updates, and deletes) and invokes the function with updated rows. Further information on the Azure SQL binding for Azure Functions is also available in the [Azure Functions docs](https://docs.microsoft.com/azure/azure-functions/functions-bindings-azure-sql). @@ -26,6 +27,7 @@ Further information on the Azure SQL binding for Azure Functions is also availab - [Tutorials](#tutorials) - [Input Binding Tutorial](#input-binding-tutorial) - [Output Binding Tutorial](#output-binding-tutorial) + - [Trigger Binding Tutorial](#trigger-binding-tutorial) - [More Samples](#more-samples) - [Input Binding](#input-binding) - [Query String](#query-string) @@ -38,6 +40,9 @@ Further information on the Azure SQL binding for Azure Functions is also availab - [Array](#array) - [Single Row](#single-row) - [Primary Keys and Identity Columns](#primary-keys-and-identity-columns) + - [Trigger Binding](#trigger-binding) + - [Change Tracking](#change-tracking) + - [Trigger Samples](#trigger-samples) - [Known Issues](#known-issues) - [Telemetry](#telemetry) - [Trademarks](#trademarks) @@ -85,6 +90,18 @@ ALTER TABLE ['{table_name}'] ALTER COLUMN ['{primary_key_column_name}'] int  ALTER TABLE ['{table_name}'] ADD CONSTRAINT PKey PRIMARY KEY CLUSTERED (['{primary_key_column_name}']); ``` +3. If you plan to use the trigger support, you need to enable [change tracking](https://docs.microsoft.com/sql/relational-databases/track-changes/about-change-tracking-sql-server) on the SQL database and the SQL table. Please note that enabling change tracking will add to the cost of the SQL server. + +```sql +ALTER DATABASE ['your database name'] +SET CHANGE_TRACKING = ON +(CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON) + +ALTER TABLE ['your table name'] +ENABLE CHANGE_TRACKING +WITH (TRACK_COLUMNS_UPDATED = ON) +``` + ### Create .NET Function App @@ -158,7 +175,7 @@ Once you have your Function App you need to configure it for use with Azure SQL } ``` -1. You have setup your local environment and are now ready to create your first SQL bindings! Continue to the [input](#Input-Binding-Tutorial) and [output](#Output-Binding-Tutorial) binding tutorials, or refer to [More Samples](#More-Samples) for information on how to use the bindings and explore on your own. +1. You have setup your local environment and are now ready to create your first SQL bindings! Continue to the [input](#Input-Binding-Tutorial), [output](#Output-Binding-Tutorial) and [trigger](#trigger-binding-tutorial) binding tutorials, or refer to [More Samples](#More-Samples) for information on how to use the bindings and explore on your own. ## Tutorials @@ -256,6 +273,46 @@ Note: This tutorial requires that a SQL database is setup as shown in [Create a - Hit 'F5' to run your code. Click the link to upsert the output array values in your SQL table. Your upserted values should launch in the browser. - Congratulations! You have successfully created your first SQL output binding! Checkout [Output Binding](#Output-Binding) for more information on how to use it and explore on your own! +### Trigger Binding Tutorial + +Note: This tutorial requires that a SQL database is setup as shown in [Create a SQL Server](#create-a-sql-server), and that you have the 'Employee.cs' file from the [Input Binding Tutorial](#input-binding-tutorial). + +- Create a new file with the following content: + + ```csharp + using System.Collections.Generic; + using Microsoft.Azure.WebJobs; + using Microsoft.Extensions.Logging; + using Microsoft.Azure.WebJobs.Extensions.Sql; + + namespace Company.Function + { + public static class EmployeeTrigger + { + [FunctionName("EmployeeTrigger")] + public static void Run( + [SqlTrigger("[dbo].[Employees]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> changes, + ILogger logger) + { + foreach (var change in changes) + { + Employee employee = change.Item; + logger.LogInformation($"Change operation: {change.Operation}"); + logger.LogInformation($"EmployeeID: {employee.EmployeeId}, FirstName: {employee.FirstName}, LastName: {employee.LastName}, Company: {employee.Company}, Team: {employee.Team}"); + } + } + } + } + ``` + +- *Skip these steps if you have not completed the output binding tutorial.* + - Open your output binding file and modify some of the values. For example, change the value of Team column from 'Functions' to 'Azure SQL'. + - Hit 'F5' to run your code. Click the link of the HTTP trigger from the output binding tutorial. +- Update, insert, or delete rows in your SQL table while the function app is running and observe the function logs. +- You should see the new log messages in the Visual Studio Code terminal containing the values of row-columns after the update operation. +- Congratulations! You have successfully created your first SQL trigger binding! Checkout [Trigger Samples](#trigger-samples) for more information on how to use it and explore on your own! + ## More Samples ### Input Binding @@ -401,8 +458,6 @@ public static async Task Run( The output binding takes a list of rows to be upserted into a user table. If the primary key value of the row already exists in the table, the row is interpreted as an update, meaning that the values of the other columns in the table for that primary key are updated. If the primary key value does not exist in the table, the row is interpreted as an insert. The upserting of the rows is batched by the output binding code. - > **NOTE:** By default the Output binding uses the T-SQL [MERGE](https://docs.microsoft.com/sql/t-sql/statements/merge-transact-sql) statement which requires [SELECT](https://docs.microsoft.com/sql/t-sql/statements/merge-transact-sql#permissions) permissions on the target database. - The output binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/SqlAttribute.cs): - **CommandText**: Passed as a constructor argument to the binding. Represents the name of the table into which rows will be upserted. @@ -524,6 +579,64 @@ This changes if one of the primary key columns is an identity column though. In 1. If the identity column isn't included in the output object then a straight insert is always performed with the other column values. See [AddProductWithIdentityColumn](./samples/samples-csharp/OutputBindingSamples/AddProductWithIdentityColumn.cs) for an example. 2. If the identity column is included (even if it's an optional nullable value) then a merge is performed similar to what happens when no identity column is present. This merge will either insert a new row or update an existing row based on the existence of a row that matches the primary keys (including the identity column). See [AddProductWithIdentityColumnIncluded](./samples/samples-csharp/OutputBindingSamples/AddProductWithIdentityColumnIncluded.cs) for an example. +### Trigger Binding + +#### Change Tracking + +The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sql/relational-databases/track-changes/about-change-tracking-sql-server) functionality to monitor the user table for changes. As such, it is necessary to enable change tracking on the SQL database and the SQL table before using the trigger support. The change tracking can be enabled through the following two queries. + +1. Enabling change tracking on the SQL database: + + ```sql + ALTER DATABASE ['your database name'] + SET CHANGE_TRACKING = ON + (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON) + ``` + + The `CHANGE_RETENTION` option specifies the duration for which the changes are retained in the change tracking table. This may affect the trigger functionality. For example, if the user application is turned off for several days and then resumed, it will only be able to catch the changes that occurred in past two days with the above query. Hence, please update the value of `CHANGE_RETENTION` to suit your requirements. The `AUTO_CLEANUP` option is used to enable or disable the clean-up task that removes the stale data. Please refer to SQL Server documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-database) for more information. + +1. Enabling change tracking on the SQL table: + + ```sql + ALTER TABLE dbo.Employees + ENABLE CHANGE_TRACKING + WITH (TRACK_COLUMNS_UPDATED = ON) + ``` + + The `TRACK_COLUMNS_UPDATED` option lets the SQL server to store information about which table columns were updated. At present, the trigger binding does not use of this information, though that functionality can be added in future. For more information, please refer to the documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-table). + + The trigger needs to have read access on the table being monitored for changes as well as to the change tracking system tables. It also needs write access to an `az_func` schema within the database, where it will create additional worker tables to store the trigger states and leases. Each function trigger will thus have an associated change tracking table and worker table. + +#### Trigger Samples +The trigger binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/TriggerBinding/SqlTriggerAttribute.cs) + +- **TableName**: Passed as a constructor argument to the binding. Represents the name of the table to be monitored for changes. +- **ConnectionStringSetting**: Specifies the name of the app setting that contains the SQL connection string used to connect to a database. The connection string must follow the format specified [here](https://docs.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring?view=sqlclient-dotnet-core-2.0). + +The trigger binding can bind to type `IReadOnlyList>`: + +- **IReadOnlyList>**: If there are multiple rows updated in the SQL table, the user function will get invoked with a batch of changes, where each element is a `SqlChange` object. Here 'T' is a generic type-argument that can be substituted with a user-defined POCO, or Plain Old C# Object, representing the user table row. The POCO should therefore follow the schema of the queried table. See the [Query String](#query-string) section for an example of what the POCO should look like. The two properties of class `SqlChange` are `Item` of type `T` which represents the table row and `Operation` of type `SqlChangeOperation` which indicates the kind of row operation (insert, update, or delete) that triggered the user function. + +Note that for insert and update operations, the user function receives POCO object containing the latest values of table columns. For delete operation, only the properties corresponding to the primary keys of the row are populated. + +Any time when the changes happen to the "Products" table, the user function will be invoked with a batch of changes. The changes are processed sequentially, so if there are a large number of changes pending to be processed, the function will be passed a batch containing the earliest changes first. + +```csharp +[FunctionName("ProductsTrigger")] +public static void Run( + [SqlTrigger("Products", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> changes, + ILogger logger) +{ + foreach (var change in changes) + { + Product product = change.Item; + logger.LogInformation($"Change occurred to Products table row: {change.Operation}"); + logger.LogInformation($"ProductID: {product.ProductID}, Name: {product.Name}, Price: {product.Cost}"); + } +} +``` + ## Known Issues - Output bindings against tables with columns of data types `NTEXT`, `TEXT`, or `IMAGE` are not supported and data upserts will fail. These types [will be removed](https://docs.microsoft.com/sql/t-sql/data-types/ntext-text-and-image-transact-sql) in a future version of SQL Server and are not compatible with the `OPENJSON` function used by this Azure Functions binding. diff --git a/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs b/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs new file mode 100644 index 000000000..4fa5cb1a7 --- /dev/null +++ b/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples +{ + public static class ProductsTrigger + { + [FunctionName("ProductsTrigger")] + public static void Run( + [SqlTrigger("[dbo].[Products]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> changes, + ILogger logger) + { + foreach (SqlChange change in changes) + { + Product product = change.Item; + logger.LogInformation($"Change occurred to Products table row: {change.Operation}"); + logger.LogInformation($"ProductID: {product.ProductID}, Name: {product.Name}, Cost: {product.Cost}"); + } + } + } +} diff --git a/samples/samples-csharp/TriggerBindingSamples/ProductsWithMultiplePrimaryColumnsTrigger.cs b/samples/samples-csharp/TriggerBindingSamples/ProductsWithMultiplePrimaryColumnsTrigger.cs new file mode 100644 index 000000000..d280da048 --- /dev/null +++ b/samples/samples-csharp/TriggerBindingSamples/ProductsWithMultiplePrimaryColumnsTrigger.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples +{ + public class MultiplePrimaryKeyProduct + { + public int ProductID { get; set; } + + public int ExternalID { get; set; } + + public string Name { get; set; } + + public int Cost { get; set; } + } + public static class ProductsWithMultiplePrimaryColumnsTrigger + { + [FunctionName("ProductsWithMultiplePrimaryColumnsTrigger")] + public static void Run( + [SqlTrigger("[dbo].[ProductsWithMultiplePrimaryColumnsAndIdentity]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> changes, + ILogger logger) + { + foreach (SqlChange change in changes) + { + MultiplePrimaryKeyProduct product = change.Item; + logger.LogInformation($"Change occurred to ProductsWithMultiplePrimaryColumns table row: {change.Operation}"); + logger.LogInformation($"ProductID: {product.ProductID}, ExternalID: {product.ExternalID} Name: {product.Name}, Cost: {product.Cost}"); + } + } + } +} \ No newline at end of file diff --git a/src/SqlBindingConfigProvider.cs b/src/SqlBindingConfigProvider.cs index 46216d076..cc366cd38 100644 --- a/src/SqlBindingConfigProvider.cs +++ b/src/SqlBindingConfigProvider.cs @@ -8,6 +8,7 @@ using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry; using Microsoft.Azure.WebJobs.Host.Bindings; using Microsoft.Azure.WebJobs.Host.Config; +using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Azure.WebJobs.Logging; @@ -21,6 +22,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql internal class SqlBindingConfigProvider : IExtensionConfigProvider { private readonly IConfiguration _configuration; + private readonly IHostIdProvider _hostIdProvider; private readonly ILoggerFactory _loggerFactory; /// @@ -29,9 +31,10 @@ internal class SqlBindingConfigProvider : IExtensionConfigProvider /// /// Thrown if either parameter is null /// - public SqlBindingConfigProvider(IConfiguration configuration, ILoggerFactory loggerFactory) + public SqlBindingConfigProvider(IConfiguration configuration, IHostIdProvider hostIdProvider, ILoggerFactory loggerFactory) { this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this._hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); this._loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); } @@ -57,6 +60,9 @@ public void Initialize(ExtensionConfigContext context) inputOutputRule.BindToInput(typeof(SqlGenericsConverter), this._configuration, logger); inputOutputRule.BindToCollector(typeof(SqlAsyncCollectorBuilder<>), this._configuration, logger); inputOutputRule.BindToInput(typeof(SqlGenericsConverter<>), this._configuration, logger); + + FluentBindingRule triggerRule = context.AddBindingRule(); + triggerRule.BindToTrigger(new SqlTriggerAttributeBindingProvider(this._configuration, this._hostIdProvider, this._loggerFactory)); } } diff --git a/src/TriggerBinding/SqlChange.cs b/src/TriggerBinding/SqlChange.cs new file mode 100644 index 000000000..c8d5430f9 --- /dev/null +++ b/src/TriggerBinding/SqlChange.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + /// + /// Represents a row that was changed in the user's table as well as metadata related to that change. + /// If the row was deleted, then is populated only with the primary key values of the deleted row + /// + /// A user-defined POCO that represents a row of the table + public sealed class SqlChange + { + /// + /// Initializes a new instance of the class. + /// + /// + /// The type of change this item corresponds to. + /// + /// + /// The current item in the user's table corresponding to the change (and only the primary key values of the row + /// in the case that it was deleted). + /// + + public SqlChange(SqlChangeOperation operation, T item) + { + this.Operation = operation; + this.Item = item; + } + + /// + /// Specifies the type of change that occurred to the row. + /// + public SqlChangeOperation Operation { get; } + + /// + /// A copy of the row that was updated/inserted in the user's table. + /// In the case that the row no longer exists in the user's table, Data is only populated with the primary key values + /// of the deleted row. + /// + public T Item { get; } + } + + public enum SqlChangeOperation + { + Insert, + Update, + Delete + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs new file mode 100644 index 000000000..3a586bdb0 --- /dev/null +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -0,0 +1,774 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Globalization; +using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using System.Text; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + /// + /// Periodically polls SQL's change table to determine if any new changes have occurred to a user's table. + /// + /// + /// Note that there is no possiblity of SQL injection in the raw queries we generate. All parameters that involve + /// inserting data from a user table are sanitized. All other parameters are generated exclusively using information + /// about the user table's schema (such as primary key column names), data stored in SQL's internal change table, or + /// data stored in our own worker table. + /// + /// A user-defined POCO that represents a row of the user's table + internal sealed class SqlTableChangeMonitor : IDisposable + { + public const int BatchSize = 10; + public const int MaxAttemptCount = 5; + public const int MaxLeaseRenewalCount = 5; + public const int LeaseIntervalInSeconds = 30; + public const int PollingIntervalInSeconds = 5; + + private readonly string _connectionString; + private readonly int _userTableId; + private readonly SqlObject _userTable; + private readonly string _userFunctionId; + private readonly string _workerTableName; + private readonly IReadOnlyList _userTableColumns; + private readonly IReadOnlyList _primaryKeyColumns; + private readonly IReadOnlyList _rowMatchConditions; + private readonly ITriggeredFunctionExecutor _executor; + private readonly ILogger _logger; + + private readonly CancellationTokenSource _cancellationTokenSourceCheckForChanges; + private readonly CancellationTokenSource _cancellationTokenSourceRenewLeases; + private CancellationTokenSource _cancellationTokenSourceExecutor; + + // It should be impossible for multiple threads to access these at the same time because of the semaphore we use. + private readonly SemaphoreSlim _rowsLock; + private IReadOnlyList> _rows; + private int _leaseRenewalCount; + private State _state = State.CheckingForChanges; + + /// + /// Initializes a new instance of the > class. + /// + /// The SQL connection string used to connect to the user's database + /// The OBJECT_ID of the user table whose changes are being tracked on + /// The name of the user table + /// The unique ID that identifies user function + /// The name of the worker table + /// List of all column names in the user table + /// List of primary key column names in the user table + /// Used to execute the user's function when changes are detected on "table" + /// Ilogger used to log information and warnings + public SqlTableChangeMonitor( + string connectionString, + int userTableId, + SqlObject userTable, + string userFunctionId, + string workerTableName, + IReadOnlyList userTableColumns, + IReadOnlyList primaryKeyColumns, + ITriggeredFunctionExecutor executor, + ILogger logger) + { + _ = !string.IsNullOrEmpty(connectionString) ? true : throw new ArgumentNullException(nameof(connectionString)); + _ = !string.IsNullOrEmpty(userTable.FullName) ? true : throw new ArgumentNullException(nameof(userTable)); + _ = !string.IsNullOrEmpty(userFunctionId) ? true : throw new ArgumentNullException(nameof(userFunctionId)); + _ = !string.IsNullOrEmpty(workerTableName) ? true : throw new ArgumentNullException(nameof(workerTableName)); + _ = userTableColumns ?? throw new ArgumentNullException(nameof(userTableColumns)); + _ = primaryKeyColumns ?? throw new ArgumentNullException(nameof(primaryKeyColumns)); + _ = executor ?? throw new ArgumentNullException(nameof(executor)); + _ = logger ?? throw new ArgumentNullException(nameof(logger)); + + this._connectionString = connectionString; + this._userTableId = userTableId; + this._userTable = userTable; + this._userFunctionId = userFunctionId; + this._workerTableName = workerTableName; + this._userTableColumns = primaryKeyColumns.Concat(userTableColumns.Except(primaryKeyColumns)).ToList(); + this._primaryKeyColumns = primaryKeyColumns; + + // Prep search-conditions that will be used besides WHERE clause to match table rows. + this._rowMatchConditions = Enumerable.Range(0, BatchSize) + .Select(index => string.Join(" AND ", primaryKeyColumns.Select(col => $"{col} = @{col}_{index}"))) + .ToList(); + + this._executor = executor; + this._logger = logger; + + this._cancellationTokenSourceCheckForChanges = new CancellationTokenSource(); + this._cancellationTokenSourceRenewLeases = new CancellationTokenSource(); + this._cancellationTokenSourceExecutor = new CancellationTokenSource(); + + this._rowsLock = new SemaphoreSlim(1); + this._rows = new List>(); + this._leaseRenewalCount = 0; + this._state = State.CheckingForChanges; + +#pragma warning disable CS4014 // Queue the below tasks and exit. Do not wait for their completion. + _ = Task.Run(() => + { + this.RunChangeConsumptionLoopAsync(); + this.RunLeaseRenewalLoopAsync(); + }); +#pragma warning restore CS4014 + } + + /// + /// Stops the change monitor which stops polling for changes on the user's table. If the change monitor is + /// currently executing a set of changes, it is only stopped once execution is finished and the user's function + /// is triggered (whether or not the trigger is successful). + /// + public void Stop() + { + this._cancellationTokenSourceCheckForChanges.Cancel(); + } + + public void Dispose() + { + throw new NotImplementedException(); + } + + /// + /// Executed once every period. If the state of the change monitor is + /// , then the method query the change/worker tables for changes on the + /// user's table. If any are found, the state of the change monitor is transitioned to + /// and the user's function is executed with the found changes. If the + /// execution is successful, the leases on "_rows" are released and the state transitions to + /// once again. + /// + private async Task RunChangeConsumptionLoopAsync() + { + try + { + CancellationToken token = this._cancellationTokenSourceCheckForChanges.Token; + + using (var connection = new SqlConnection(this._connectionString)) + { + await connection.OpenAsync(token); + + while (!token.IsCancellationRequested) + { + if (this._state == State.CheckingForChanges) + { + // What should we do if this call gets stuck? + await this.GetChangesAsync(token); + await this.ProcessChangesAsync(token); + } + + await Task.Delay(TimeSpan.FromSeconds(PollingIntervalInSeconds), token); + } + } + } + catch (Exception e) + { + // Only want to log the exception if it wasn't caused by StopAsync being called, since Task.Delay + // throws an exception if it's cancelled. + if (e.GetType() != typeof(TaskCanceledException)) + { + this._logger.LogError(e.Message); + } + } + finally + { + // If this thread exits due to any reason, then the lease renewal thread should exit as well. Otherwise, + // it will keep looping perpetually. + this._cancellationTokenSourceRenewLeases.Cancel(); + this._cancellationTokenSourceCheckForChanges.Dispose(); + this._cancellationTokenSourceExecutor.Dispose(); + } + } + + /// + /// Queries the change/worker tables to check for new changes on the user's table. If any are found, stores the + /// change along with the corresponding data from the user table in "_rows". + /// + private async Task GetChangesAsync(CancellationToken token) + { + try + { + using (var connection = new SqlConnection(this._connectionString)) + { + await connection.OpenAsync(token); + + using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + { + try + { + // Update the version number stored in the global state table if necessary before using it. + using (SqlCommand updateTablesPreInvocationCommand = this.BuildUpdateTablesPreInvocation(connection, transaction)) + { + await updateTablesPreInvocationCommand.ExecuteNonQueryAsync(token); + } + + // Use the version number to query for new changes. + using (SqlCommand getChangesCommand = this.BuildGetChangesCommand(connection, transaction)) + { + var rows = new List>(); + using (SqlDataReader reader = await getChangesCommand.ExecuteReaderAsync(token)) + { + while (await reader.ReadAsync(token)) + { + rows.Add(SqlBindingUtilities.BuildDictionaryFromSqlRow(reader)); + } + } + + this._rows = rows; + } + + // If changes were found, acquire leases on them. + if (this._rows.Count > 0) + { + using (SqlCommand acquireLeasesCommand = this.BuildAcquireLeasesCommand(connection, transaction)) + { + await acquireLeasesCommand.ExecuteNonQueryAsync(token); + } + } + transaction.Commit(); + } + catch (Exception ex) + { + this._logger.LogError("Commit Exception Type: {0}", ex.GetType()); + this._logger.LogError(" Message: {0}", ex.Message); + try + { + transaction.Rollback(); + } + catch (Exception ex2) + { + this._logger.LogError("Rollback Exception Type: {0}", ex2.GetType()); + this._logger.LogError(" Message: {0}", ex2.Message); + } + } + } + } + } + catch (Exception e) + { + // If there's an exception in any part of the process, we want to clear all of our data in memory and + // retry checking for changes again. + this._rows = new List>(); + this._logger.LogWarning($"Failed to check {this._userTable.FullName} for new changes due to error: {e.Message}"); + } + } + + private async Task ProcessChangesAsync(CancellationToken token) + { + if (this._rows.Count > 0) + { + this._state = State.ProcessingChanges; + IReadOnlyList> changes = null; + + try + { + // What should we do if this fails? It doesn't make sense to retry since it's not a connection based + // thing. We could still try to trigger on the correctly processed changes, but that adds additional + // complication because we don't want to release the leases on the incorrectly processed changes. + // For now, just give up I guess? + changes = this.GetChanges(); + } + catch (Exception e) + { + await this.ClearRowsAsync( + $"Failed to extract user table data from table {this._userTable.FullName} associated " + + $"with change metadata due to error: {e.Message}", true); + } + + if (changes != null) + { + FunctionResult result = await this._executor.TryExecuteAsync( + new TriggeredFunctionData() { TriggerValue = changes }, + this._cancellationTokenSourceExecutor.Token); + + if (result.Succeeded) + { + await this.ReleaseLeasesAsync(token); + } + else + { + // In the future might make sense to retry executing the function, but for now we just let + // another worker try. + await this.ClearRowsAsync( + $"Failed to trigger user's function for table {this._userTable.FullName} due to " + + $"error: {result.Exception.Message}", true); + } + } + } + } + + /// + /// Executed once every period. If the state of the change monitor is + /// , then we will renew the leases held by the change monitor on "_rows". + /// + private async void RunLeaseRenewalLoopAsync() + { + try + { + CancellationToken token = this._cancellationTokenSourceRenewLeases.Token; + + using (var connection = new SqlConnection(this._connectionString)) + { + await connection.OpenAsync(token); + + while (!token.IsCancellationRequested) + { + await this._rowsLock.WaitAsync(token); + + await this.RenewLeasesAsync(connection, token); + + // Want to make sure to renew the leases before they expire, so we renew them twice per lease period. + await Task.Delay(TimeSpan.FromSeconds(LeaseIntervalInSeconds / 2), token); + } + } + } + catch (Exception e) + { + // Only want to log the exception if it wasn't caused by StopAsync being called, since Task.Delay throws + // an exception if it's cancelled. + if (e.GetType() != typeof(TaskCanceledException)) + { + this._logger.LogError(e.Message); + } + } + finally + { + this._cancellationTokenSourceRenewLeases.Dispose(); + } + } + + private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken token) + { + try + { + if (this._state == State.ProcessingChanges) + { + // I don't think I need a transaction for renewing leases. If this worker reads in a row from the + // worker table and determines that it corresponds to its batch of changes, but then that row gets + // deleted by a cleanup task, it shouldn't renew the lease on it anyways. + using (SqlCommand renewLeasesCommand = this.BuildRenewLeasesCommand(connection)) + { + await renewLeasesCommand.ExecuteNonQueryAsync(token); + } + } + } + catch (Exception e) + { + // This catch block is necessary so that the finally block is executed even in the case of an exception + // (see https://docs.microsoft.com/dotnet/csharp/language-reference/keywords/try-finally, third + // paragraph). If we fail to renew the leases, multiple workers could be processing the same change + // data, but we have functionality in place to deal with this (see design doc). + this._logger.LogError($"Failed to renew leases due to error: {e.Message}"); + } + finally + { + if (this._state == State.ProcessingChanges) + { + // Do we want to update this count even in the case of a failure to renew the leases? Probably, + // because the count is simply meant to indicate how much time the other thread has spent processing + // changes essentially. + this._leaseRenewalCount += 1; + + // If this thread has been cancelled, then the _cancellationTokenSourceExecutor could have already + // been disposed so shouldn't cancel it. + if (this._leaseRenewalCount == MaxLeaseRenewalCount && !token.IsCancellationRequested) + { + this._logger.LogWarning("Call to execute the function (TryExecuteAsync) seems to be stuck, so it is being cancelled"); + + // If we keep renewing the leases, the thread responsible for processing the changes is stuck. + // If it's stuck, it has to be stuck in the function execution call (I think), so we should + // cancel the call. + this._cancellationTokenSourceExecutor.Cancel(); + this._cancellationTokenSourceExecutor = new CancellationTokenSource(); + } + } + + // Want to always release the lock at the end, even if renewing the leases failed. + this._rowsLock.Release(); + } + } + + /// + /// Resets the in-memory state of the change monitor and sets it to start polling for changes again. + /// + /// + /// The error messages the logger will report describing the reason function execution failed (used only in the case of a failure). + /// + /// True if ClearRowsAsync should acquire the "_rowsLock" (only true in the case of a failure) + private async Task ClearRowsAsync(string error, bool acquireLock) + { + if (acquireLock) + { + this._logger.LogError(error); + await this._rowsLock.WaitAsync(); + } + + this._leaseRenewalCount = 0; + this._state = State.CheckingForChanges; + this._rows = new List>(); + this._rowsLock.Release(); + } + + /// + /// Releases the leases held on "_rows". + /// + /// + private async Task ReleaseLeasesAsync(CancellationToken token) + { + // Don't want to change the "_rows" while another thread is attempting to renew leases on them. + await this._rowsLock.WaitAsync(token); + long newLastSyncVersion = this.RecomputeLastSyncVersion(); + + try + { + using (var connection = new SqlConnection(this._connectionString)) + { + await connection.OpenAsync(token); + using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + { + try + { + // Release the leases held on "_rows". + using (SqlCommand releaseLeasesCommand = this.BuildReleaseLeasesCommand(connection, transaction)) + { + await releaseLeasesCommand.ExecuteNonQueryAsync(token); + } + + // Update the global state table if we have processed all changes with ChangeVersion <= newLastSyncVersion, + // and clean up the worker table to remove all rows with ChangeVersion <= newLastSyncVersion. + using (SqlCommand updateTablesPostInvocationCommand = this.BuildUpdateTablesPostInvocation(connection, transaction, newLastSyncVersion)) + { + await updateTablesPostInvocationCommand.ExecuteNonQueryAsync(token); + } + + transaction.Commit(); + } + catch (Exception ex) + { + this._logger.LogError("Commit Exception Type: {0}", ex.GetType()); + this._logger.LogError(" Message: {0}", ex.Message); + try + { + transaction.Rollback(); + } + catch (Exception ex2) + { + this._logger.LogError("Rollback Exception Type: {0}", ex2.GetType()); + this._logger.LogError(" Message: {0}", ex2.Message); + } + } + } + } + + } + catch (Exception e) + { + // What should we do if releasing the leases fails? We could try to release them again or just wait, + // since eventually the lease time will expire. Then another thread will re-process the same changes + // though, so less than ideal. But for now that's the functionality. + this._logger.LogError($"Failed to release leases for user table {this._userTable.FullName} due to error: {e.Message}"); + } + finally + { + // Want to do this before releasing the lock in case the renew leases thread wakes up. It will see that + // the state is checking for changes and not renew the (just released) leases. + await this.ClearRowsAsync(string.Empty, false); + } + } + + /// + /// Calculates the new version number to attempt to update LastSyncVersion in global state table to. If all + /// version numbers in _rows are the same, use that version number. If they aren't, use the second largest + /// version number. For an explanation as to why this method was chosen, see 9c in Steps of Operation in this + /// design doc: https://microsoft-my.sharepoint.com/:w:/p/t-sotevo/EQdANWq9ZWpKm8e48TdzUwcBGZW07vJmLf8TL_rtEG8ixQ?e=owN2EX. + /// + private long RecomputeLastSyncVersion() + { + var changeVersionSet = new SortedSet(); + foreach (IReadOnlyDictionary row in this._rows) + { + string changeVersion = row["SYS_CHANGE_VERSION"]; + changeVersionSet.Add(long.Parse(changeVersion, CultureInfo.InvariantCulture)); + } + + // If there are at least two version numbers in this set, return the second highest one. Otherwise, return + // the only version number in the set. + return changeVersionSet.ElementAt(changeVersionSet.Count > 1 ? changeVersionSet.Count - 2 : 0); + } + + /// + /// Builds up the list of passed to the user's triggered function based on the data + /// stored in "_rows". If any of the changes correspond to a deleted row, then the + /// will be populated with only the primary key values of the deleted row. + /// + /// The list of changes + private IReadOnlyList> GetChanges() + { + var changes = new List>(); + foreach (IReadOnlyDictionary row in this._rows) + { + SqlChangeOperation operation = GetChangeOperation(row); + + // If the row has been deleted, there is no longer any data for it in the user table. The best we can do + // is populate the row-item with the primary key values of the row. + Dictionary item = operation == SqlChangeOperation.Delete + ? this._primaryKeyColumns.ToDictionary(col => col, col => row[col]) + : this._userTableColumns.ToDictionary(col => col, col => row[col]); + + changes.Add(new SqlChange(operation, JsonConvert.DeserializeObject(JsonConvert.SerializeObject(item)))); + } + + return changes; + } + + /// + /// Gets the change associated with this row (either an insert, update or delete). + /// + /// The (combined) row from the change table and worker table + /// Thrown if the value of the "SYS_CHANGE_OPERATION" column is none of "I", "U", or "D" + /// SqlChangeOperation.Insert for an insert, SqlChangeOperation.Update for an update, and SqlChangeOperation.Delete for a delete + private static SqlChangeOperation GetChangeOperation(IReadOnlyDictionary row) + { + string operation = row["SYS_CHANGE_OPERATION"]; + switch (operation) + { + case "I": return SqlChangeOperation.Insert; + case "U": return SqlChangeOperation.Update; + case "D": return SqlChangeOperation.Delete; + default: throw new InvalidDataException($"Invalid change type encountered in change table row: {row}"); + }; + } + + /// + /// Builds the command to update the global state table in the case of a new minimum valid version number. + /// Sets the LastSyncVersion for this _userTable to be the new minimum valid version number. + /// + /// The connection to add to the returned SqlCommand + /// The transaction to add to the returned SqlCommand + /// The SqlCommand populated with the query and appropriate parameters + private SqlCommand BuildUpdateTablesPreInvocation(SqlConnection connection, SqlTransaction transaction) + { + string updateTablesPreInvocationQuery = $@" + DECLARE @min_valid_version bigint; + SET @min_valid_version = CHANGE_TRACKING_MIN_VALID_VERSION({this._userTableId}); + + DECLARE @last_sync_version bigint; + SELECT @last_sync_version = LastSyncVersion + FROM {SqlTriggerConstants.GlobalStateTableName} + WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; + + IF @last_sync_version < @min_valid_version + UPDATE {SqlTriggerConstants.GlobalStateTableName} + SET LastSyncVersion = @min_valid_version + WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; + "; + + return new SqlCommand(updateTablesPreInvocationQuery, connection, transaction); + } + + /// + /// Builds the query to check for changes on the user's table (). + /// + /// The connection to add to the returned SqlCommand + /// The transaction to add to the returned SqlCommand + /// The SqlCommand populated with the query and appropriate parameters + private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransaction transaction) + { + string selectList = string.Join(", ", this._userTableColumns.Select(col => this._primaryKeyColumns.Contains(col) ? $"c.{col}" : $"u.{col}")); + string userTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col} = u.{col}")); + string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col} = w.{col}")); + + string getChangesQuery = $@" + DECLARE @last_sync_version bigint; + SELECT @last_sync_version = LastSyncVersion + FROM {SqlTriggerConstants.GlobalStateTableName} + WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; + + SELECT TOP {BatchSize} + {selectList}, + c.SYS_CHANGE_VERSION, c.SYS_CHANGE_OPERATION, + w.ChangeVersion, w.AttemptCount, w.LeaseExpirationTime + FROM CHANGETABLE (CHANGES {this._userTable.FullName}, @last_sync_version) AS c + LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} + LEFT OUTER JOIN {this._userTable.FullName} AS u ON {userTableJoinCondition} + WHERE + (w.LeaseExpirationTime IS NULL AND (w.ChangeVersion IS NULL OR w.ChangeVersion < c.SYS_CHANGE_VERSION) OR + w.LeaseExpirationTime < SYSDATETIME()) AND + (w.AttemptCount IS NULL OR w.AttemptCount < {MaxAttemptCount}) + ORDER BY c.SYS_CHANGE_VERSION ASC; + "; + + return new SqlCommand(getChangesQuery, connection, transaction); + } + + /// + /// Builds the query to acquire leases on the rows in "_rows" if changes are detected in the user's table + /// (). + /// + /// The connection to add to the returned SqlCommand + /// The transaction to add to the returned SqlCommand + /// The SqlCommand populated with the query and appropriate parameters + private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransaction transaction) + { + var acquireLeasesQuery = new StringBuilder(); + + for (int index = 0; index < this._rows.Count; index++) + { + string valuesList = string.Join(", ", this._primaryKeyColumns.Select(col => $"@{col}_{index}")); + string changeVersion = this._rows[index]["SYS_CHANGE_VERSION"]; + + acquireLeasesQuery.Append($@" + IF NOT EXISTS (SELECT * FROM {this._workerTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[index]}) + INSERT INTO {this._workerTableName} WITH (TABLOCKX) + VALUES ({valuesList}, {changeVersion}, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME())); + ELSE + UPDATE {this._workerTableName} WITH (TABLOCKX) + SET + ChangeVersion = {changeVersion}, + AttemptCount = AttemptCount + 1, + LeaseExpirationTime = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + WHERE {this._rowMatchConditions[index]}; + "); + } + + return this.GetSqlCommandWithParameters(acquireLeasesQuery.ToString(), connection, transaction); + } + + /// + /// Builds the query to renew leases on the rows in "_rows" (). + /// + /// The connection to add to the returned SqlCommand + /// The SqlCommand populated with the query and appropriate parameters + private SqlCommand BuildRenewLeasesCommand(SqlConnection connection) + { + var renewLeasesQuery = new StringBuilder(); + + for (int index = 0; index < this._rows.Count; index++) + { + renewLeasesQuery.Append($@" + UPDATE {this._workerTableName} WITH (TABLOCKX) + SET LeaseExpirationTime = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + WHERE {this._rowMatchConditions[index]}; + "); + } + + return this.GetSqlCommandWithParameters(renewLeasesQuery.ToString(), connection, null); + } + + /// + /// Builds the query to release leases on the rows in "_rows" after successful invocation of the user's function + /// (). + /// + /// The connection to add to the returned SqlCommand + /// The transaction to add to the returned SqlCommand + /// The SqlCommand populated with the query and appropriate parameters + private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransaction transaction) + { + var releaseLeasesQuery = new StringBuilder("DECLARE @current_change_version bigint;\n"); + + for (int index = 0; index < this._rows.Count; index++) + { + string changeVersion = this._rows[index]["SYS_CHANGE_VERSION"]; + + releaseLeasesQuery.Append($@" + SELECT @current_change_version = ChangeVersion + FROM {this._workerTableName} WITH (TABLOCKX) + WHERE {this._rowMatchConditions[index]}; + + IF @current_change_version <= {changeVersion} + UPDATE {this._workerTableName} WITH (TABLOCKX) + SET ChangeVersion = {changeVersion}, AttemptCount = 0, LeaseExpirationTime = NULL + WHERE {this._rowMatchConditions[index]}; + "); + } + + return this.GetSqlCommandWithParameters(releaseLeasesQuery.ToString(), connection, transaction); + } + + /// + /// Builds the command to update the global version number in _globalStateTable after successful invocation of + /// the user's function. If the global version number is updated, also cleans the worker table and removes all + /// rows for which ChangeVersion <= newLastSyncVersion. + /// + /// The connection to add to the returned SqlCommand + /// The transaction to add to the returned SqlCommand + /// The new LastSyncVersion to store in the _globalStateTable for this _userTableName + /// The SqlCommand populated with the query and appropriate parameters + private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, SqlTransaction transaction, long newLastSyncVersion) + { + string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col} = w.{col}")); + + // TODO: Need to think through all cases to ensure the query below is correct, especially with use of < vs <=. + string updateTablesPostInvocationQuery = $@" + DECLARE @current_last_sync_version bigint; + SELECT @current_last_sync_version = LastSyncVersion + FROM {SqlTriggerConstants.GlobalStateTableName} + WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; + + DECLARE @unprocessed_changes bigint; + SELECT @unprocessed_changes = COUNT(*) FROM ( + SELECT c.SYS_CHANGE_VERSION + FROM CHANGETABLE(CHANGES {this._userTable.FullName}, @current_last_sync_version) AS c + LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} + WHERE + c.SYS_CHANGE_VERSION <= {newLastSyncVersion} AND + ((w.ChangeVersion IS NULL OR w.ChangeVersion != c.SYS_CHANGE_VERSION OR w.LeaseExpirationTime IS NOT NULL) AND + (w.AttemptCount IS NULL OR w.AttemptCount < {MaxAttemptCount}))) AS Changes + + IF @unprocessed_changes = 0 AND @current_last_sync_version < {newLastSyncVersion} + BEGIN + UPDATE {SqlTriggerConstants.GlobalStateTableName} + SET LastSyncVersion = {newLastSyncVersion} + WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; + + DELETE FROM {this._workerTableName} WITH (TABLOCKX) WHERE ChangeVersion <= {newLastSyncVersion}; + END + "; + + return new SqlCommand(updateTablesPostInvocationQuery, connection, transaction); + } + + /// + /// Returns SqlCommand with SqlParameters added to it. Each parameter follows the format + /// (@PrimaryKey_i, PrimaryKeyValue), where @PrimaryKey is the name of a primary key column, and PrimaryKeyValue + /// is one of the row's value for that column. To distinguish between the parameters of different rows, each row + /// will have a distinct value of i. + /// + /// SQL query string + /// The connection to add to the returned SqlCommand + /// The transaction to add to the returned SqlCommand + /// + /// Ideally, we would have a map that maps from rows to a list of SqlCommands populated with their primary key + /// values. The issue with this is that SQL doesn't seem to allow adding parameters to one collection when they + /// are part of another. So, for example, since the SqlParameters are part of the list in the map, an exception + /// is thrown if they are also added to the collection of a SqlCommand. The expected behavior seems to be to + /// rebuild the SqlParameters each time. + /// + private SqlCommand GetSqlCommandWithParameters(string commandText, SqlConnection connection, SqlTransaction transaction) + { + var command = new SqlCommand(commandText, connection, transaction); + + for (int index = 0; index < this._rows.Count; index++) + { + foreach (string col in this._primaryKeyColumns) + { + command.Parameters.Add(new SqlParameter($"@{col}_{index}", this._rows[index][col])); + } + } + + return command; + } + + private enum State + { + CheckingForChanges, + ProcessingChanges, + } + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerAttribute.cs b/src/TriggerBinding/SqlTriggerAttribute.cs new file mode 100644 index 000000000..02ebcb074 --- /dev/null +++ b/src/TriggerBinding/SqlTriggerAttribute.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.WebJobs.Description; + +namespace Microsoft.Azure.WebJobs +{ + /// + /// A trigger binding that can be used to establish a connection to a SQL server database and trigger a user's function + /// whenever changes happen to a given table in that database + /// + [Binding] + [AttributeUsage(AttributeTargets.Parameter)] + public sealed class SqlTriggerAttribute : Attribute + { + /// + /// Initializes a new instance of the class. + /// + /// The name of the table to monitor for changes + public SqlTriggerAttribute(string tableName) + { + this.TableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); + } + + /// + /// The name of the app setting where the SQL connection string is stored + /// (see https://docs.microsoft.com/en-us/dotnet/api/microsoft.data.sqlclient.sqlconnection?view=sqlclient-dotnet-core-2.0). + /// The attributes specified in the connection string are listed here + /// https://docs.microsoft.com/en-us/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring?view=sqlclient-dotnet-core-2.0 + /// For example, to create a connection to the "TestDB" located at the URL "test.database.windows.net" using a User ID and password, + /// create a ConnectionStringSetting with a name like SqlServerAuthentication. The value of the SqlServerAuthentication app setting + /// would look like "Data Source=test.database.windows.net;Database=TestDB;User ID={userid};Password={password}". + /// + public string ConnectionStringSetting { get; set; } + + /// + /// The name of the table to monitor for changes + /// + public string TableName { get; } + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerAttributeBindingProvider.cs b/src/TriggerBinding/SqlTriggerAttributeBindingProvider.cs new file mode 100644 index 000000000..7c6f2b81e --- /dev/null +++ b/src/TriggerBinding/SqlTriggerAttributeBindingProvider.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Triggers; +using Microsoft.Azure.WebJobs.Logging; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + internal sealed class SqlTriggerAttributeBindingProvider : ITriggerBindingProvider + { + private readonly IConfiguration _configuration; + private readonly IHostIdProvider _hostIdProvider; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// + /// Used to extract the connection string from connectionStringSetting + /// + /// + /// Used to fetch a unique host identifier + /// + /// + /// Used to create a logger for the SQL trigger binding + /// + /// + /// Thrown if either parameter is null + /// + public SqlTriggerAttributeBindingProvider(IConfiguration configuration, IHostIdProvider hostIdProvider, ILoggerFactory loggerFactory) + { + this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this._hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); + _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + this._logger = loggerFactory.CreateLogger(LogCategories.CreateTriggerCategory("Sql")); + } + + /// + /// Creates a SqlTriggerBinding using the information provided in "context" + /// + /// + /// Contains the SqlTriggerAttribute used to build up a SqlTriggerBinding + /// + /// + /// Thrown if context is null + /// + /// + /// If the SqlTriggerAttribute is bound to an invalid Type. Currently only IReadOnlyList> + /// is supported, where T is a user-defined POCO representing a row of their table + /// + /// + /// Null if "context" does not contain a SqlTriggerAttribute. Otherwise returns a SqlTriggerBinding{T} associated + /// with the SqlTriggerAttribute in "context", where T is the user-defined POCO + /// + public Task TryCreateAsync(TriggerBindingProviderContext context) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + ParameterInfo parameter = context.Parameter; + SqlTriggerAttribute attribute = parameter.GetCustomAttribute(inherit: false); + + if (attribute == null) + { + return Task.FromResult(default(ITriggerBinding)); + } + + if (!IsValidType(parameter.ParameterType)) + { + throw new InvalidOperationException($"Can't bind SqlTriggerAttribute to type {parameter.ParameterType}." + + " Only IReadOnlyList> is supported, where T is a user-defined POCO that matches the" + + " schema of the tracked table"); + } + + string connectionString = SqlBindingUtilities.GetConnectionString(attribute.ConnectionStringSetting, this._configuration); + + Type type = parameter.ParameterType.GetGenericArguments()[0].GetGenericArguments()[0]; + Type typeOfTriggerBinding = typeof(SqlTriggerBinding<>).MakeGenericType(type); + ConstructorInfo constructor = typeOfTriggerBinding.GetConstructor( + new Type[] { typeof(string), typeof(string), typeof(ParameterInfo), typeof(IHostIdProvider), typeof(ILogger) }); + + return Task.FromResult((ITriggerBinding)constructor.Invoke( + new object[] { attribute.TableName, connectionString, parameter, this._hostIdProvider, this._logger })); + } + + /// + /// Determines if type is a valid Type, Currently only IReadOnlyList> is supported, where T is a + /// user-defined POCO representing a row of their table. + /// + /// + /// True is type is a valid Type, otherwise false + private static bool IsValidType(Type type) + { + return + type.IsGenericType && + type.GetGenericTypeDefinition() == typeof(IReadOnlyList<>) && + type.GetGenericArguments()[0].IsGenericType && + type.GetGenericArguments()[0].GetGenericTypeDefinition() == typeof(SqlChange<>); + } + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerBinding.cs b/src/TriggerBinding/SqlTriggerBinding.cs new file mode 100644 index 000000000..1ceb2e3a8 --- /dev/null +++ b/src/TriggerBinding/SqlTriggerBinding.cs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Linq; +using Microsoft.Azure.WebJobs.Host.Bindings; +using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Listeners; +using Microsoft.Azure.WebJobs.Host.Protocols; +using Microsoft.Azure.WebJobs.Host.Triggers; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + /// + /// Represents the SQL trigger binding for a given user table being monitored for changes + /// + /// A user-defined POCO that represents a row of the user's table + internal sealed class SqlTriggerBinding : ITriggerBinding + { + private readonly string _connectionString; + private readonly string _tableName; + private readonly ParameterInfo _parameter; + private readonly IHostIdProvider _hostIdProvider; + private readonly ILogger _logger; + private static readonly IReadOnlyDictionary _emptyBindingContract = new Dictionary(); + + /// + /// Initializes a new instance of the class. + /// + /// + /// The name of the user table that changes are being tracked on + /// + /// + /// The SQL connection string used to connect to the user's database + /// + /// + /// The parameter that contains the SqlTriggerAttribute of the user's function + /// + /// + /// Used to fetch a unique host identifier + /// + /// + /// Thrown if any of the parameters are null + /// + public SqlTriggerBinding(string tableName, string connectionString, ParameterInfo parameter, IHostIdProvider hostIdProvider, ILogger logger) + { + this._tableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); + this._connectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString)); + this._parameter = parameter ?? throw new ArgumentNullException(nameof(parameter)); + this._hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); + this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// Gets the type of the value the Trigger receives from the Executor. + /// + public Type TriggerValueType => typeof(IReadOnlyList>); + + /// + /// Returns an empty binding contract. The type that SqlTriggerAttribute is bound to is checked in + /// + /// + public IReadOnlyDictionary BindingDataContract => _emptyBindingContract; + + /// + /// Binds the list of represented by "value" with a + /// which (as the name suggests) simply returns "value". + /// + /// The list of data. + /// + /// + /// Unused + /// + /// + /// Thrown if "value" is not of type IReadOnlyList>. + /// + /// + /// The ITriggerData which stores the list of SQL table changes as well as the SimpleValueBinder + /// + public Task BindAsync(object value, ValueBindingContext context) + { + if (!(value is IReadOnlyList> changes)) + { + throw new InvalidOperationException("The value passed to the SqlTrigger BindAsync must be of type IReadOnlyList>"); + } + + var bindingData = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "SqlTrigger", changes } + }; + + return Task.FromResult(new TriggerData(new SimpleValueProvider(this._parameter.ParameterType, changes, this._tableName), bindingData)); + } + + /// + /// Creates a listener that will monitor for changes to the user's table + /// + /// + /// Context for the listener, including the executor that executes the user's function when changes are detected in the user's table + /// + /// + /// Thrown if context is null + /// + /// + /// The listener + /// + public async Task CreateListenerAsync(ListenerFactoryContext context) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context), "Missing listener context"); + } + + string userFunctionId = SqlBindingUtilities.AsSingleQuoteEscapedString(await this.GetUserFunctionIdAsync()); + return new SqlTriggerListener(this._connectionString, this._tableName, userFunctionId, context.Executor, this._logger); + } + + /// A description of the SqlTriggerParameter ( + public ParameterDescriptor ToParameterDescriptor() + { + return new SqlTriggerParameterDescriptor + { + Name = this._parameter.Name, + Type = "SqlTrigger", + TableName = _tableName + }; + } + + /// + /// Creates a unique ID for user function using host ID and method name. + /// + /// We call the WebJobs SDK library method to generate the host ID. The host ID is essentially a hash of the + /// assembly name containing the user function(s). This ensures that if the user ever updates their application, + /// unless the assembly name is modified, the new application version will be able to resume from the point + /// where the previous version had left. Appending another hash of class+method in here ensures that if there + /// are multiple user functions within the same process and tracking the same SQL table, then each one of them + /// gets a separate view of the table changes. + /// + private async Task GetUserFunctionIdAsync() + { + string hostId = await this._hostIdProvider.GetHostIdAsync(CancellationToken.None); + + var methodInfo = (MethodInfo)this._parameter.Member; + string functionName = $"{methodInfo.DeclaringType.FullName}.{methodInfo.Name}"; + + using (var sha256 = SHA256.Create()) + { + byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(hostId + functionName)); + + return SqlBindingUtilities.AsSingleQuoteEscapedString(new Guid(hash.Take(16).ToArray()).ToString("N").Substring(0, 16)); + } + } + + /// + /// Simply returns whatever value was passed to it in the constructor without modifying it + /// + internal class SimpleValueProvider : IValueProvider + { + private readonly object _value; + private readonly string _invokeString; + + public SimpleValueProvider(Type type, object value, string invokeString) + { + this.Type = type; + this._value = value; + this._invokeString = invokeString; + } + + /// + /// Returns the type that the trigger binding is bound to (IReadOnlyList{SqlChange{T}}"/>>) + /// + public Type Type { get; } + + public Task GetValueAsync() + { + return Task.FromResult(this._value); + } + + /// + /// Returns the table name that changes are being tracked on + /// + /// + public string ToInvokeString() + { + return this._invokeString; + } + } + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs new file mode 100644 index 000000000..6aa13358d --- /dev/null +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + internal static class SqlTriggerConstants + { + public const string SchemaName = "az_func"; + + public const string GlobalStateTableName = "[" + SchemaName + "].[GlobalState]"; + + public const string WorkerTableNameFormat = "[" + SchemaName + "].[Worker_{0}]"; + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs new file mode 100644 index 000000000..b7fbe73c5 --- /dev/null +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Listeners; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + /// A user-defined POCO that represents a row of the user's table + internal sealed class SqlTriggerListener : IListener + { + private readonly SqlObject _userTable; + private readonly string _connectionString; + private readonly string _userFunctionId; + private readonly ITriggeredFunctionExecutor _executor; + private readonly ILogger _logger; + + private SqlTableChangeMonitor _changeMonitor; + private State _state; + + /// + /// Initializes a new instance of the > + /// + /// The SQL connection string used to connect to the user's database + /// The name of the user table whose changes are being tracked on + /// + /// The unique ID that identifies user function. If multiple application instances are executing the same user + /// function, they are all supposed to have the same user function ID. + /// + /// Used to execute the user's function when changes are detected on "table" + /// Ilogger used to log information and warnings + public SqlTriggerListener(string connectionString, string tableName, string userFunctionId, ITriggeredFunctionExecutor executor, ILogger logger) + { + _ = !string.IsNullOrEmpty(connectionString) ? true : throw new ArgumentNullException(nameof(connectionString)); + _ = !string.IsNullOrEmpty(tableName) ? true : throw new ArgumentNullException(nameof(tableName)); + _ = !string.IsNullOrEmpty(userFunctionId) ? true : throw new ArgumentNullException(nameof(userFunctionId)); + _ = executor ?? throw new ArgumentNullException(nameof(executor)); + _ = logger ?? throw new ArgumentNullException(nameof(logger)); + + this._connectionString = connectionString; + this._userTable = new SqlObject(tableName); + this._userFunctionId = userFunctionId; + this._executor = executor; + this._logger = logger; + this._state = State.NotInitialized; + } + + /// + /// Stops the listener which stops checking for changes on the user's table. + /// + public void Cancel() + { + this.StopAsync(CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Disposes resources held by the listener to poll for changes. + /// + public void Dispose() + { + this.StopAsync(CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Starts the listener if it has not yet been started, which starts polling for changes on the user's table. + /// + /// The cancellation token + public async Task StartAsync(CancellationToken cancellationToken) + { + if (this._state == State.NotInitialized) + { + using (var connection = new SqlConnection(this._connectionString)) + { + await connection.OpenAsync(cancellationToken); + + int userTableId = await this.GetUserTableIdAsync(connection, cancellationToken); + IReadOnlyList<(string name, string type)> primaryKeyColumns = await this.GetPrimaryKeyColumnsAsync(connection, cancellationToken); + IReadOnlyList userTableColumns = await this.GetUserTableColumnsAsync(connection, cancellationToken); + + string workerTableName = string.Format(CultureInfo.InvariantCulture, SqlTriggerConstants.WorkerTableNameFormat, $"{this._userFunctionId}_{userTableId}"); + + using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + { + await CreateSchemaAsync(connection, transaction, cancellationToken); + await CreateGlobalStateTableAsync(connection, transaction, cancellationToken); + await this.InsertGlobalStateTableRowAsync(connection, transaction, userTableId, cancellationToken); + await CreateWorkerTablesAsync(connection, transaction, workerTableName, primaryKeyColumns, cancellationToken); + transaction.Commit(); + } + + // TODO: Check if passing cancellation token would be beneficial. + this._changeMonitor = new SqlTableChangeMonitor( + this._connectionString, + userTableId, + this._userTable, + this._userFunctionId, + workerTableName, + userTableColumns, + primaryKeyColumns.Select(col => col.name).ToList(), + this._executor, + this._logger); + + this._state = State.Running; + } + } + } + + /// + /// Stops the listener (if it was started), which stops checking for changes on the user's table. + /// + public Task StopAsync(CancellationToken cancellationToken) + { + // Nothing to stop if the change monitor has either already been stopped or hasn't been started. + if (this._state == State.Running) + { + this._changeMonitor.Stop(); + this._state = State.Stopped; + } + return Task.CompletedTask; + } + + /// + /// Returns the OBJECT_ID of userTable + /// + /// + /// Thrown if the query to retrieve the OBJECT_ID of the user table fails to correctly execute + /// This can happen if the OBJECT_ID call returns NULL, meaning that the user table might not exist in the database + /// + private async Task GetUserTableIdAsync(SqlConnection connection, CancellationToken cancellationToken) + { + string getObjectIdQuery = $"SELECT OBJECT_ID(N{this._userTable.QuotedName}, 'U');"; + + using (var getObjectIdCommand = new SqlCommand(getObjectIdQuery, connection)) + { + using (SqlDataReader reader = await getObjectIdCommand.ExecuteReaderAsync(cancellationToken)) + { + + // TODO: Check if the below if-block ever gets hit. + if (!await reader.ReadAsync(cancellationToken)) + { + throw new InvalidOperationException($"Failed to determine the OBJECT_ID of the user table {this._userTable.FullName}"); + } + + object userTableId = reader.GetValue(0); + + if (userTableId is DBNull) + { + throw new InvalidOperationException($"Failed to determine the OBJECT_ID of the user table {this._userTable.FullName}. " + + "Possibly the table does not exist in the database."); + } + + return (int)userTableId; + } + } + } + + /// + /// Gets the names and types of primary key columns of the user's table. + /// + /// + /// Thrown if no primary keys are found for the user table. This could be because the user table does not have + /// any primary key columns. + /// + private async Task> GetPrimaryKeyColumnsAsync(SqlConnection connection, CancellationToken cancellationToken) + { + string getPrimaryKeyColumnsQuery = $@" + SELECT c.name, t.name, c.max_length, c.precision, c.scale + FROM sys.indexes AS i + INNER JOIN sys.index_columns AS ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id + INNER JOIN sys.columns AS c ON ic.object_id = c.object_id AND ic.column_id = c.column_id + INNER JOIN sys.types AS t ON c.user_type_id = t.user_type_id + WHERE i.is_primary_key = 1 AND i.object_id = OBJECT_ID(N{this._userTable.QuotedName}, 'U'); + "; + + using (var getPrimaryKeyColumnsCommand = new SqlCommand(getPrimaryKeyColumnsQuery, connection)) + { + using (SqlDataReader reader = await getPrimaryKeyColumnsCommand.ExecuteReaderAsync(cancellationToken)) + { + + string[] variableLengthTypes = new string[] { "varchar", "nvarchar", "nchar", "char", "binary", "varbinary" }; + string[] variablePrecisionTypes = new string[] { "numeric", "decimal" }; + + var primaryKeyColumns = new List<(string name, string type)>(); + + while (await reader.ReadAsync(cancellationToken)) + { + string type = reader.GetString(1); + + if (variableLengthTypes.Contains(type)) + { + // Special "max" case. I'm actually not sure it's valid to have varchar(max) as a primary key because + // it exceeds the byte limit of an index field (900 bytes), but just in case + short length = reader.GetInt16(2); + type += length == -1 ? "(max)" : $"({length})"; + } + else if (variablePrecisionTypes.Contains(type)) + { + byte precision = reader.GetByte(3); + byte scale = reader.GetByte(4); + type += $"({precision},{scale})"; + } + + primaryKeyColumns.Add((name: reader.GetString(0), type)); + } + + if (primaryKeyColumns.Count == 0) + { + throw new InvalidOperationException($"Unable to determine the primary keys of user table {this._userTable.FullName}. " + + "Potentially, the table does not have any primary key columns. A primary key is required for every " + + "user table for which changes are being monitored."); + } + + return primaryKeyColumns; + } + } + } + + /// + /// Gets the column names of the user's table. + /// + private async Task> GetUserTableColumnsAsync(SqlConnection connection, CancellationToken cancellationToken) + { + string getUserTableColumnsQuery = $@" + SELECT name + FROM sys.columns + WHERE object_id = OBJECT_ID(N{this._userTable.QuotedName}, 'U'); + "; + + using (var getUserTableColumnsCommand = new SqlCommand(getUserTableColumnsQuery, connection)) + { + using (SqlDataReader reader = await getUserTableColumnsCommand.ExecuteReaderAsync(cancellationToken)) + { + + var userTableColumns = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + userTableColumns.Add(reader.GetString(0)); + } + + return userTableColumns; + } + } + } + + /// + /// Creates the schema where the worker tables will be located if it does not already exist. + /// + private static async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) + { + string createSchemaQuery = $@" + IF SCHEMA_ID(N'{SqlTriggerConstants.SchemaName}') IS NULL + EXEC ('CREATE SCHEMA [{SqlTriggerConstants.SchemaName}]'); + "; + + using (var createSchemaCommand = new SqlCommand(createSchemaQuery, connection, transaction)) + { + await createSchemaCommand.ExecuteNonQueryAsync(cancellationToken); + } + } + + /// + /// Creates the global state table if it does not already exist. + /// + private static async Task CreateGlobalStateTableAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) + { + string createGlobalStateTableQuery = $@" + IF OBJECT_ID(N'{SqlTriggerConstants.GlobalStateTableName}', 'U') IS NULL + CREATE TABLE {SqlTriggerConstants.GlobalStateTableName} ( + UserFunctionID char(16) NOT NULL, + UserTableID int NOT NULL, + LastSyncVersion bigint NOT NULL, + PRIMARY KEY (UserFunctionID, UserTableID) + ); + "; + + using (var createGlobalStateTableCommand = new SqlCommand(createGlobalStateTableQuery, connection, transaction)) + { + await createGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); + } + } + + /// + /// Inserts row for the user table and the function inside the global state table, if one does not already exist. + /// + private async Task InsertGlobalStateTableRowAsync(SqlConnection connection, SqlTransaction transaction, int userTableId, CancellationToken cancellationToken) + { + string insertRowGlobalStateTableQuery = $@" + IF NOT EXISTS ( + SELECT * FROM {SqlTriggerConstants.GlobalStateTableName} + WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {userTableId} + ) + INSERT INTO {SqlTriggerConstants.GlobalStateTableName} + VALUES ('{this._userFunctionId}', {userTableId}, CHANGE_TRACKING_MIN_VALID_VERSION({userTableId})); + "; + + using (var insertRowGlobalStateTableCommand = new SqlCommand(insertRowGlobalStateTableQuery, connection, transaction)) + { + try + { + await insertRowGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); + } + catch (Exception e) + { + // Could fail if we try to insert a NULL value into the LastSyncVersion, which happens when + // CHANGE_TRACKING_MIN_VALID_VERSION returns NULL for the user table, meaning that change tracking is + // not enabled for either the database or table (or both). + + string errorMessage = $"Failed to start processing changes to table {this._userTable.FullName}, " + + $"potentially because change tracking was not enabled for the table or database {connection.Database}."; + + this._logger.LogWarning(errorMessage + $" Exact exception thrown is {e.Message}"); + throw new InvalidOperationException(errorMessage); + } + } + } + + /// + /// Creates the worker table associated with the user's table, if one does not already exist. + /// + private static async Task CreateWorkerTablesAsync( + SqlConnection connection, + SqlTransaction transaction, + string workerTableName, + IReadOnlyList<(string name, string type)> primaryKeyColumns, + CancellationToken cancellationToken) + { + string primaryKeysWithTypes = string.Join(", ", primaryKeyColumns.Select(col => $"{col.name} {col.type}")); + string primaryKeys = string.Join(", ", primaryKeyColumns.Select(col => col.name)); + + // TODO: Check if some of the table columns below can have 'NOT NULL' property. + string createWorkerTableQuery = $@" + IF OBJECT_ID(N'{workerTableName}', 'U') IS NULL + CREATE TABLE {workerTableName} ( + {primaryKeysWithTypes}, + ChangeVersion bigint NOT NULL, + AttemptCount int NOT NULL, + LeaseExpirationTime datetime2, + PRIMARY KEY ({primaryKeys}) + ); + "; + + using (var createWorkerTableCommand = new SqlCommand(createWorkerTableQuery, connection, transaction)) + { + await createWorkerTableCommand.ExecuteNonQueryAsync(cancellationToken); + } + } + + private enum State + { + NotInitialized, + Running, + Stopped, + } + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerParameterDescriptor.cs b/src/TriggerBinding/SqlTriggerParameterDescriptor.cs new file mode 100644 index 000000000..20d37160a --- /dev/null +++ b/src/TriggerBinding/SqlTriggerParameterDescriptor.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Host.Protocols; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + internal sealed class SqlTriggerParameterDescriptor : TriggerParameterDescriptor + { + /// + /// Name of the table being monitored + /// + public string TableName { get; set; } + + /// + /// The reason the user's function was triggered. Specifies the table name that experienced changes + /// as well as the time the changes were detected + /// + /// Unused + /// A string with the reason + public override string GetTriggerReason(IDictionary arguments) + { + return $"New changes on table {this.TableName} at {DateTime.UtcNow:o}"; + } + } +} \ No newline at end of file diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index fd2ee1b3d..3baddfae9 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -23,7 +23,7 @@ public class IntegrationTestBase : IDisposable /// /// Host process for Azure Function CLI /// - private Process FunctionHost; + public Process FunctionHost { get; set; } /// /// Host process for Azurite local storage emulator. This is required for non-HTTP trigger functions: @@ -139,6 +139,24 @@ private void ExecuteAllScriptsInFolder(string folder) } } + protected void EnableChangeTracking() + { + string enableChangeTrackingDatabaseQuery = $@"ALTER DATABASE [{this.DatabaseName}] + SET CHANGE_TRACKING = ON + (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON); + "; + this.ExecuteNonQuery(enableChangeTrackingDatabaseQuery); + string enableChangeTrackingTableQuery = $@" + ALTER TABLE [Products] + ENABLE CHANGE_TRACKING + WITH (TRACK_COLUMNS_UPDATED = ON); + ALTER TABLE [ProductsWithMultiplePrimaryColumnsAndIdentity] + ENABLE CHANGE_TRACKING + WITH (TRACK_COLUMNS_UPDATED = ON); + "; + this.ExecuteNonQuery(enableChangeTrackingTableQuery); + } + /// /// This starts the Azurite storage emulator. /// @@ -324,17 +342,6 @@ public void Dispose() { this.TestOutput.WriteLine($"Failed to close connection. Error: {e1.Message}"); } - try - { - // Drop the test database - using var masterConnection = new SqlConnection(this.MasterConnectionString); - masterConnection.Open(); - TestUtils.ExecuteNonQuery(masterConnection, $"DROP DATABASE IF EXISTS {this.DatabaseName}"); - } - catch (Exception e2) - { - this.TestOutput.WriteLine($"Failed to drop {this.DatabaseName}, Error: {e2.Message}"); - } finally { this.Connection.Dispose(); @@ -345,9 +352,9 @@ public void Dispose() this.FunctionHost?.Kill(); this.FunctionHost?.Dispose(); } - catch (Exception e3) + catch (Exception e2) { - this.TestOutput.WriteLine($"Failed to stop function host, Error: {e3.Message}"); + this.TestOutput.WriteLine($"Failed to stop function host, Error: {e2.Message}"); } try @@ -355,9 +362,20 @@ public void Dispose() this.AzuriteHost?.Kill(); this.AzuriteHost?.Dispose(); } + catch (Exception e3) + { + this.TestOutput.WriteLine($"Failed to stop Azurite, Error: {e3.Message}"); + } + try + { + // Drop the test database + using var masterConnection = new SqlConnection(this.MasterConnectionString); + masterConnection.Open(); + TestUtils.ExecuteNonQuery(masterConnection, $"DROP DATABASE IF EXISTS {this.DatabaseName}"); + } catch (Exception e4) { - this.TestOutput.WriteLine($"Failed to stop Azurite, Error: {e4.Message}"); + this.TestOutput.WriteLine($"Failed to drop {this.DatabaseName}, Error: {e4.Message}"); } } GC.SuppressFinalize(this); diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs new file mode 100644 index 000000000..429d8edc9 --- /dev/null +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Diagnostics; +using System.Text; +using System.Linq; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration +{ + [Collection("IntegrationTests")] + public class SqlTriggerBindingIntegrationTests : IntegrationTestBase + { + public SqlTriggerBindingIntegrationTests(ITestOutputHelper output) : base(output) + { + } + + /// + /// Tests for insertion of products triggering the function. + /// + [Fact] + public async void InsertProductsTest() + { + int countInsert = 0; + this.EnableChangeTracking(); + this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); + this.FunctionHost.OutputDataReceived += (object sender, DataReceivedEventArgs e) => + { + if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to Products table row")) + { + countInsert++; + } + }; + + Product[] products = GetProducts(3, 100); + this.InsertProducts(products); + await Task.Delay(5000); + + Assert.Equal(3, countInsert); + + } + + /// + /// Tests insertion into table with multiple primary key columns. + /// + [Fact] + public void InsertMultiplePrimaryKeyColumnsTest() + { + this.EnableChangeTracking(); + this.StartFunctionHost(nameof(ProductsWithMultiplePrimaryColumnsTrigger), Common.SupportedLanguages.CSharp); + var taskCompletionSource = new TaskCompletionSource(); + this.FunctionHost.OutputDataReceived += (object sender, DataReceivedEventArgs e) => + { + if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to ProductsWithMultiplePrimaryColumns table row")) + { + taskCompletionSource.SetResult(true); + } + }; + + string query = $@"INSERT INTO dbo.ProductsWithMultiplePrimaryColumnsAndIdentity VALUES(123, 'ProductTest', 100);"; + this.ExecuteNonQuery(query); + taskCompletionSource.Task.Wait(10000); + + Assert.True(taskCompletionSource.Task.Result); + + } + + /// + /// Tests for behaviour of the trigger when insertion, updates, and deletes occur. + /// + [Fact] + public async void InsertUpdateDeleteProductsTest() + { + int countInsert = 0; + int countUpdate = 0; + int countDelete = 0; + this.EnableChangeTracking(); + this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); + this.FunctionHost.OutputDataReceived += (object sender, DataReceivedEventArgs e) => + { + if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to Products table row: Insert")) + { + countInsert++; + } + if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to Products table row: Update")) + { + countUpdate++; + } + if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to Products table row: Delete")) + { + countDelete++; + } + }; + Product[] products = GetProducts(3, 100); + this.InsertProducts(products); + await Task.Delay(500); + this.UpdateProducts(products.Take(2).ToArray()); + await Task.Delay(500); + this.DeleteProducts(products.Take(1).ToArray()); + + await Task.Delay(5000); + + //Since insert and update counts as a single insert and insert and delete counts as a single delete + Assert.Equal(2, countInsert); + Assert.Equal(0, countUpdate); + Assert.Equal(1, countDelete); + + } + + private static Product[] GetProducts(int n, int cost) + { + var result = new Product[n]; + for (int i = 1; i <= n; i++) + { + result[i - 1] = new Product + { + ProductID = i, + Name = "test", + Cost = cost * i + }; + } + return result; + } + private void InsertProducts(Product[] products) + { + if (products.Length == 0) + { + return; + } + + var queryBuilder = new StringBuilder(); + foreach (Product p in products) + { + queryBuilder.AppendLine($"INSERT INTO dbo.Products VALUES({p.ProductID}, '{p.Name}', {p.Cost});"); + } + + this.ExecuteNonQuery(queryBuilder.ToString()); + } + private void UpdateProducts(Product[] products) + { + if (products.Length == 0) + { + return; + } + + var queryBuilder = new StringBuilder(); + foreach (Product p in products) + { + string newName = p.Name + "Update"; + queryBuilder.AppendLine($"UPDATE dbo.Products set Name = '{newName}' where ProductId = {p.ProductID};"); + } + + this.ExecuteNonQuery(queryBuilder.ToString()); + } + private void DeleteProducts(Product[] products) + { + if (products.Length == 0) + { + return; + } + + var queryBuilder = new StringBuilder(); + foreach (Product p in products) + { + queryBuilder.AppendLine($"DELETE from dbo.Products where ProductId = {p.ProductID};"); + } + + this.ExecuteNonQuery(queryBuilder.ToString()); + } + } +} \ No newline at end of file diff --git a/test/Unit/SqlInputBindingTests.cs b/test/Unit/SqlInputBindingTests.cs index feab46991..0e6701dc5 100644 --- a/test/Unit/SqlInputBindingTests.cs +++ b/test/Unit/SqlInputBindingTests.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading; using static Microsoft.Azure.WebJobs.Extensions.Sql.SqlConverters; +using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -18,6 +19,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Unit public class SqlInputBindingTests { private static readonly Mock config = new Mock(); + private static readonly Mock hostIdProvider = new Mock(); private static readonly Mock loggerFactory = new Mock(); private static readonly Mock logger = new Mock(); private static readonly SqlConnection connection = new SqlConnection(); @@ -25,8 +27,9 @@ public class SqlInputBindingTests [Fact] public void TestNullConfiguration() { - Assert.Throws(() => new SqlBindingConfigProvider(null, loggerFactory.Object)); - Assert.Throws(() => new SqlBindingConfigProvider(config.Object, null)); + Assert.Throws(() => new SqlBindingConfigProvider(null, hostIdProvider.Object, loggerFactory.Object)); + Assert.Throws(() => new SqlBindingConfigProvider(config.Object, null, loggerFactory.Object)); + Assert.Throws(() => new SqlBindingConfigProvider(config.Object, hostIdProvider.Object, null)); Assert.Throws(() => new SqlConverter(null)); Assert.Throws(() => new SqlGenericsConverter(null, logger.Object)); } @@ -40,7 +43,7 @@ public void TestNullCommandText() [Fact] public void TestNullContext() { - var configProvider = new SqlBindingConfigProvider(config.Object, loggerFactory.Object); + var configProvider = new SqlBindingConfigProvider(config.Object, hostIdProvider.Object, loggerFactory.Object); Assert.Throws(() => configProvider.Initialize(null)); } diff --git a/test/Unit/SqlTriggerBindingTests.cs b/test/Unit/SqlTriggerBindingTests.cs new file mode 100644 index 000000000..348076286 --- /dev/null +++ b/test/Unit/SqlTriggerBindingTests.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Reflection; +using System.Collections.Generic; +using System.Threading; +using Microsoft.Extensions.Configuration; +using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Triggers; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using Microsoft.Extensions.Logging; +using Xunit; +using Moq; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Unit +{ + public class TriggerBindingTests + { + private static readonly Mock config = new Mock(); + private static readonly Mock hostIdProvider = new Mock(); + private static readonly Mock loggerFactory = new Mock(); + private static readonly Mock mockExecutor = new Mock(); + private static readonly Mock logger = new Mock(); + private static readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + + [Fact] + public void TestTriggerBindingProviderNullConfig() + { + Assert.Throws(() => new SqlTriggerAttributeBindingProvider(null, hostIdProvider.Object, loggerFactory.Object)); + Assert.Throws(() => new SqlTriggerAttributeBindingProvider(config.Object, null, loggerFactory.Object)); + Assert.Throws(() => new SqlTriggerAttributeBindingProvider(config.Object, hostIdProvider.Object, null)); + } + + [Fact] + public async void TestTriggerAttributeBindingProviderNullContext() + { + var configProvider = new SqlTriggerAttributeBindingProvider(config.Object, hostIdProvider.Object, loggerFactory.Object); + await Assert.ThrowsAsync(() => configProvider.TryCreateAsync(null)); + } + + [Fact] + public void TestTriggerListenerNullConfig() + { + string connectionString = "testConnectionString"; + string tableName = "testTableName"; + string userFunctionId = "testUserFunctionId"; + + Assert.Throws(() => new SqlTriggerListener(null, tableName, userFunctionId, mockExecutor.Object, logger.Object)); + Assert.Throws(() => new SqlTriggerListener(connectionString, null, userFunctionId, mockExecutor.Object, logger.Object)); + Assert.Throws(() => new SqlTriggerListener(connectionString, tableName, null, mockExecutor.Object, logger.Object)); + Assert.Throws(() => new SqlTriggerListener(connectionString, tableName, userFunctionId, null, logger.Object)); + Assert.Throws(() => new SqlTriggerListener(connectionString, tableName, userFunctionId, mockExecutor.Object, null)); + } + + [Fact] + public void TestTriggerBindingNullConfig() + { + string connectionString = "testConnectionString"; + string tableName = "testTableName"; + + Assert.Throws(() => new SqlTriggerBinding(null, connectionString, TriggerBindingFunctionTest.GetParamForChanges(), hostIdProvider.Object, logger.Object)); + Assert.Throws(() => new SqlTriggerBinding(tableName, null, TriggerBindingFunctionTest.GetParamForChanges(), hostIdProvider.Object, logger.Object)); + Assert.Throws(() => new SqlTriggerBinding(tableName, connectionString, null, hostIdProvider.Object, logger.Object)); + Assert.Throws(() => new SqlTriggerBinding(tableName, connectionString, TriggerBindingFunctionTest.GetParamForChanges(), null, logger.Object)); + Assert.Throws(() => new SqlTriggerBinding(tableName, connectionString, TriggerBindingFunctionTest.GetParamForChanges(), hostIdProvider.Object, null)); + } + + [Fact] + public async void TestTriggerBindingProviderWithInvalidParameter() + { + var triggerBindingProviderContext = new TriggerBindingProviderContext(TriggerBindingFunctionTest.GetParamForChanges(), cancellationTokenSource.Token); + var triggerAttributeBindingProvider = new SqlTriggerAttributeBindingProvider(config.Object, hostIdProvider.Object, loggerFactory.Object); + + //Trying to create a SqlTriggerBinding with IEnumerable> type for the changes + //This is expected to throw an exception as the type expected for receiving the changes is IReadOnlyList> + await Assert.ThrowsAsync(() => triggerAttributeBindingProvider.TryCreateAsync(triggerBindingProviderContext)); + } + + /// + /// Creating a function using trigger with wrong parameter for changes field. + /// + private static class TriggerBindingFunctionTest + { + /// + ///Example function created with wrong parameter + /// + public static void InvalidParameterType( + [SqlTrigger("[dbo].[Employees]", ConnectionStringSetting = "SqlConnectionString")] + IEnumerable> changes, + ILogger logger) + { + logger.LogInformation(changes.ToString()); + } + /// + ///Gets the parameter info for changes in the function + /// + public static ParameterInfo GetParamForChanges() + { + MethodInfo methodInfo = typeof(TriggerBindingFunctionTest).GetMethod("InvalidParameterType", BindingFlags.Public | BindingFlags.Static); + ParameterInfo[] parameters = methodInfo.GetParameters(); + return parameters[^2]; + } + } + } +} \ No newline at end of file From 6e332044f2aaea7cd6574c17bf6398a83bb020b3 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Wed, 27 Jul 2022 15:56:51 +0530 Subject: [PATCH 02/77] Add trigger binding support --- README.md | 12 +- .../TriggerBindingSamples/ProductsTrigger.cs | 9 +- ...oductsWithMultiplePrimaryColumnsTrigger.cs | 35 --- src/SqlBindingConfigProvider.cs | 2 +- src/TriggerBinding/SqlChange.cs | 26 +- src/TriggerBinding/SqlTableChangeMonitor.cs | 11 +- src/TriggerBinding/SqlTriggerAttribute.cs | 18 +- .../SqlTriggerAttributeBindingProvider.cs | 110 -------- src/TriggerBinding/SqlTriggerBinding.cs | 126 ++------- .../SqlTriggerBindingProvider.cs | 99 +++++++ src/TriggerBinding/SqlTriggerListener.cs | 124 +++++---- .../SqlTriggerParameterDescriptor.cs | 15 +- src/TriggerBinding/SqlTriggerValueProvider.cs | 50 ++++ test/Integration/IntegrationTestBase.cs | 83 +++--- .../SqlTriggerBindingIntegrationTests.cs | 243 +++++++++--------- test/Unit/SqlTriggerBindingTests.cs | 120 ++++----- 16 files changed, 464 insertions(+), 619 deletions(-) delete mode 100644 samples/samples-csharp/TriggerBindingSamples/ProductsWithMultiplePrimaryColumnsTrigger.cs delete mode 100644 src/TriggerBinding/SqlTriggerAttributeBindingProvider.cs create mode 100644 src/TriggerBinding/SqlTriggerBindingProvider.cs create mode 100644 src/TriggerBinding/SqlTriggerValueProvider.cs diff --git a/README.md b/README.md index b5cf92db6..db980867a 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ Note: This tutorial requires that a SQL database is setup as shown in [Create a IReadOnlyList> changes, ILogger logger) { - foreach (var change in changes) + foreach (SqlChange change in changes) { Employee employee = change.Item; logger.LogInformation($"Change operation: {change.Operation}"); @@ -613,9 +613,9 @@ The trigger binding takes two [arguments](https://github.com/Azure/azure-functio - **TableName**: Passed as a constructor argument to the binding. Represents the name of the table to be monitored for changes. - **ConnectionStringSetting**: Specifies the name of the app setting that contains the SQL connection string used to connect to a database. The connection string must follow the format specified [here](https://docs.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring?view=sqlclient-dotnet-core-2.0). -The trigger binding can bind to type `IReadOnlyList>`: +The trigger binding can bind to type `IReadOnlyList>`: -- **IReadOnlyList>**: If there are multiple rows updated in the SQL table, the user function will get invoked with a batch of changes, where each element is a `SqlChange` object. Here 'T' is a generic type-argument that can be substituted with a user-defined POCO, or Plain Old C# Object, representing the user table row. The POCO should therefore follow the schema of the queried table. See the [Query String](#query-string) section for an example of what the POCO should look like. The two properties of class `SqlChange` are `Item` of type `T` which represents the table row and `Operation` of type `SqlChangeOperation` which indicates the kind of row operation (insert, update, or delete) that triggered the user function. +- **IReadOnlyList>**: If there are multiple rows updated in the SQL table, the user function will get invoked with a batch of changes, where each element is a `SqlChange` object. Here `T` is a generic type-argument that can be substituted with a user-defined POCO, or Plain Old C# Object, representing the user table row. The POCO should therefore follow the schema of the queried table. See the [Query String](#query-string) section for an example of what the POCO should look like. The two properties of class `SqlChange` are `Item` of type `T` which represents the table row and `Operation` of type `SqlChangeOperation` which indicates the kind of row operation (insert, update, or delete) that triggered the user function. Note that for insert and update operations, the user function receives POCO object containing the latest values of table columns. For delete operation, only the properties corresponding to the primary keys of the row are populated. @@ -628,11 +628,11 @@ public static void Run( IReadOnlyList> changes, ILogger logger) { - foreach (var change in changes) + foreach (SqlChange change in changes) { Product product = change.Item; - logger.LogInformation($"Change occurred to Products table row: {change.Operation}"); - logger.LogInformation($"ProductID: {product.ProductID}, Name: {product.Name}, Price: {product.Cost}"); + logger.LogInformation($"Change operation: {change.Operation}"); + logger.LogInformation($"ProductID: {product.ProductID}, Name: {product.Name}, Cost: {product.Cost}"); } } ``` diff --git a/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs b/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs index 4fa5cb1a7..dedaab06d 100644 --- a/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs +++ b/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; using Microsoft.Extensions.Logging; +using Newtonsoft.Json; namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples { @@ -15,12 +16,8 @@ public static void Run( IReadOnlyList> changes, ILogger logger) { - foreach (SqlChange change in changes) - { - Product product = change.Item; - logger.LogInformation($"Change occurred to Products table row: {change.Operation}"); - logger.LogInformation($"ProductID: {product.ProductID}, Name: {product.Name}, Cost: {product.Cost}"); - } + // The output is used to inspect the trigger binding parameter in test methods. + logger.LogInformation("SQL Changes: " + JsonConvert.SerializeObject(changes)); } } } diff --git a/samples/samples-csharp/TriggerBindingSamples/ProductsWithMultiplePrimaryColumnsTrigger.cs b/samples/samples-csharp/TriggerBindingSamples/ProductsWithMultiplePrimaryColumnsTrigger.cs deleted file mode 100644 index d280da048..000000000 --- a/samples/samples-csharp/TriggerBindingSamples/ProductsWithMultiplePrimaryColumnsTrigger.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Collections.Generic; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples -{ - public class MultiplePrimaryKeyProduct - { - public int ProductID { get; set; } - - public int ExternalID { get; set; } - - public string Name { get; set; } - - public int Cost { get; set; } - } - public static class ProductsWithMultiplePrimaryColumnsTrigger - { - [FunctionName("ProductsWithMultiplePrimaryColumnsTrigger")] - public static void Run( - [SqlTrigger("[dbo].[ProductsWithMultiplePrimaryColumnsAndIdentity]", ConnectionStringSetting = "SqlConnectionString")] - IReadOnlyList> changes, - ILogger logger) - { - foreach (SqlChange change in changes) - { - MultiplePrimaryKeyProduct product = change.Item; - logger.LogInformation($"Change occurred to ProductsWithMultiplePrimaryColumns table row: {change.Operation}"); - logger.LogInformation($"ProductID: {product.ProductID}, ExternalID: {product.ExternalID} Name: {product.Name}, Cost: {product.Cost}"); - } - } - } -} \ No newline at end of file diff --git a/src/SqlBindingConfigProvider.cs b/src/SqlBindingConfigProvider.cs index cc366cd38..d8c6e08ad 100644 --- a/src/SqlBindingConfigProvider.cs +++ b/src/SqlBindingConfigProvider.cs @@ -62,7 +62,7 @@ public void Initialize(ExtensionConfigContext context) inputOutputRule.BindToInput(typeof(SqlGenericsConverter<>), this._configuration, logger); FluentBindingRule triggerRule = context.AddBindingRule(); - triggerRule.BindToTrigger(new SqlTriggerAttributeBindingProvider(this._configuration, this._hostIdProvider, this._loggerFactory)); + triggerRule.BindToTrigger(new SqlTriggerBindingProvider(this._configuration, this._hostIdProvider, this._loggerFactory)); } } diff --git a/src/TriggerBinding/SqlChange.cs b/src/TriggerBinding/SqlChange.cs index c8d5430f9..61fa53564 100644 --- a/src/TriggerBinding/SqlChange.cs +++ b/src/TriggerBinding/SqlChange.cs @@ -4,23 +4,16 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql { /// - /// Represents a row that was changed in the user's table as well as metadata related to that change. - /// If the row was deleted, then is populated only with the primary key values of the deleted row + /// Represents the changed row in the user table. /// - /// A user-defined POCO that represents a row of the table + /// POCO class representing the row in the user table public sealed class SqlChange { /// /// Initializes a new instance of the class. /// - /// - /// The type of change this item corresponds to. - /// - /// - /// The current item in the user's table corresponding to the change (and only the primary key values of the row - /// in the case that it was deleted). - /// - + /// Change operation + /// POCO representing the row in the user table on which the change operation took place public SqlChange(SqlChangeOperation operation, T item) { this.Operation = operation; @@ -28,18 +21,21 @@ public SqlChange(SqlChangeOperation operation, T item) } /// - /// Specifies the type of change that occurred to the row. + /// Change operation (insert, update, or delete). /// public SqlChangeOperation Operation { get; } /// - /// A copy of the row that was updated/inserted in the user's table. - /// In the case that the row no longer exists in the user's table, Data is only populated with the primary key values - /// of the deleted row. + /// POCO representing the row in the user table on which the change operation took place. If the change + /// operation is , then only the properties corresponding to the primary + /// keys will be populated. /// public T Item { get; } } + /// + /// Represents the type of change operation in the table row. + /// public enum SqlChangeOperation { Insert, diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 3a586bdb0..1215e9a44 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -3,16 +3,16 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; -using System.Globalization; using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using Newtonsoft.Json; -using System.Text; namespace Microsoft.Azure.WebJobs.Extensions.Sql { @@ -126,14 +126,9 @@ public SqlTableChangeMonitor( /// currently executing a set of changes, it is only stopped once execution is finished and the user's function /// is triggered (whether or not the trigger is successful). /// - public void Stop() - { - this._cancellationTokenSourceCheckForChanges.Cancel(); - } - public void Dispose() { - throw new NotImplementedException(); + this._cancellationTokenSourceCheckForChanges.Cancel(); } /// diff --git a/src/TriggerBinding/SqlTriggerAttribute.cs b/src/TriggerBinding/SqlTriggerAttribute.cs index 02ebcb074..2321dd374 100644 --- a/src/TriggerBinding/SqlTriggerAttribute.cs +++ b/src/TriggerBinding/SqlTriggerAttribute.cs @@ -7,35 +7,29 @@ namespace Microsoft.Azure.WebJobs { /// - /// A trigger binding that can be used to establish a connection to a SQL server database and trigger a user's function - /// whenever changes happen to a given table in that database + /// Attribute used to bind a parameter to SQL trigger message. /// [Binding] [AttributeUsage(AttributeTargets.Parameter)] public sealed class SqlTriggerAttribute : Attribute { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - /// The name of the table to monitor for changes + /// Name of the user table public SqlTriggerAttribute(string tableName) { this.TableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); } /// - /// The name of the app setting where the SQL connection string is stored - /// (see https://docs.microsoft.com/en-us/dotnet/api/microsoft.data.sqlclient.sqlconnection?view=sqlclient-dotnet-core-2.0). - /// The attributes specified in the connection string are listed here - /// https://docs.microsoft.com/en-us/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring?view=sqlclient-dotnet-core-2.0 - /// For example, to create a connection to the "TestDB" located at the URL "test.database.windows.net" using a User ID and password, - /// create a ConnectionStringSetting with a name like SqlServerAuthentication. The value of the SqlServerAuthentication app setting - /// would look like "Data Source=test.database.windows.net;Database=TestDB;User ID={userid};Password={password}". + /// Name of the app setting containing the SQL connection string. /// + [ConnectionString] public string ConnectionStringSetting { get; set; } /// - /// The name of the table to monitor for changes + /// Name of the user table. /// public string TableName { get; } } diff --git a/src/TriggerBinding/SqlTriggerAttributeBindingProvider.cs b/src/TriggerBinding/SqlTriggerAttributeBindingProvider.cs deleted file mode 100644 index 7c6f2b81e..000000000 --- a/src/TriggerBinding/SqlTriggerAttributeBindingProvider.cs +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Executors; -using Microsoft.Azure.WebJobs.Host.Triggers; -using Microsoft.Azure.WebJobs.Logging; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Azure.WebJobs.Extensions.Sql -{ - internal sealed class SqlTriggerAttributeBindingProvider : ITriggerBindingProvider - { - private readonly IConfiguration _configuration; - private readonly IHostIdProvider _hostIdProvider; - private readonly ILogger _logger; - - /// - /// Initializes a new instance of the class. - /// - /// - /// Used to extract the connection string from connectionStringSetting - /// - /// - /// Used to fetch a unique host identifier - /// - /// - /// Used to create a logger for the SQL trigger binding - /// - /// - /// Thrown if either parameter is null - /// - public SqlTriggerAttributeBindingProvider(IConfiguration configuration, IHostIdProvider hostIdProvider, ILoggerFactory loggerFactory) - { - this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - this._hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); - _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); - this._logger = loggerFactory.CreateLogger(LogCategories.CreateTriggerCategory("Sql")); - } - - /// - /// Creates a SqlTriggerBinding using the information provided in "context" - /// - /// - /// Contains the SqlTriggerAttribute used to build up a SqlTriggerBinding - /// - /// - /// Thrown if context is null - /// - /// - /// If the SqlTriggerAttribute is bound to an invalid Type. Currently only IReadOnlyList> - /// is supported, where T is a user-defined POCO representing a row of their table - /// - /// - /// Null if "context" does not contain a SqlTriggerAttribute. Otherwise returns a SqlTriggerBinding{T} associated - /// with the SqlTriggerAttribute in "context", where T is the user-defined POCO - /// - public Task TryCreateAsync(TriggerBindingProviderContext context) - { - if (context == null) - { - throw new ArgumentNullException(nameof(context)); - } - - ParameterInfo parameter = context.Parameter; - SqlTriggerAttribute attribute = parameter.GetCustomAttribute(inherit: false); - - if (attribute == null) - { - return Task.FromResult(default(ITriggerBinding)); - } - - if (!IsValidType(parameter.ParameterType)) - { - throw new InvalidOperationException($"Can't bind SqlTriggerAttribute to type {parameter.ParameterType}." + - " Only IReadOnlyList> is supported, where T is a user-defined POCO that matches the" + - " schema of the tracked table"); - } - - string connectionString = SqlBindingUtilities.GetConnectionString(attribute.ConnectionStringSetting, this._configuration); - - Type type = parameter.ParameterType.GetGenericArguments()[0].GetGenericArguments()[0]; - Type typeOfTriggerBinding = typeof(SqlTriggerBinding<>).MakeGenericType(type); - ConstructorInfo constructor = typeOfTriggerBinding.GetConstructor( - new Type[] { typeof(string), typeof(string), typeof(ParameterInfo), typeof(IHostIdProvider), typeof(ILogger) }); - - return Task.FromResult((ITriggerBinding)constructor.Invoke( - new object[] { attribute.TableName, connectionString, parameter, this._hostIdProvider, this._logger })); - } - - /// - /// Determines if type is a valid Type, Currently only IReadOnlyList> is supported, where T is a - /// user-defined POCO representing a row of their table. - /// - /// - /// True is type is a valid Type, otherwise false - private static bool IsValidType(Type type) - { - return - type.IsGenericType && - type.GetGenericTypeDefinition() == typeof(IReadOnlyList<>) && - type.GetGenericArguments()[0].IsGenericType && - type.GetGenericArguments()[0].GetGenericTypeDefinition() == typeof(SqlChange<>); - } - } -} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerBinding.cs b/src/TriggerBinding/SqlTriggerBinding.cs index 1ceb2e3a8..b04fd95c4 100644 --- a/src/TriggerBinding/SqlTriggerBinding.cs +++ b/src/TriggerBinding/SqlTriggerBinding.cs @@ -19,9 +19,9 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql { /// - /// Represents the SQL trigger binding for a given user table being monitored for changes + /// Represents the SQL trigger parameter binding. /// - /// A user-defined POCO that represents a row of the user's table + /// POCO class representing the row in the user table internal sealed class SqlTriggerBinding : ITriggerBinding { private readonly string _connectionString; @@ -29,112 +29,60 @@ internal sealed class SqlTriggerBinding : ITriggerBinding private readonly ParameterInfo _parameter; private readonly IHostIdProvider _hostIdProvider; private readonly ILogger _logger; + private static readonly IReadOnlyDictionary _emptyBindingContract = new Dictionary(); + private static readonly IReadOnlyDictionary _emptyBindingData = new Dictionary(); /// /// Initializes a new instance of the class. /// - /// - /// The name of the user table that changes are being tracked on - /// - /// - /// The SQL connection string used to connect to the user's database - /// - /// - /// The parameter that contains the SqlTriggerAttribute of the user's function - /// - /// - /// Used to fetch a unique host identifier - /// - /// - /// Thrown if any of the parameters are null - /// - public SqlTriggerBinding(string tableName, string connectionString, ParameterInfo parameter, IHostIdProvider hostIdProvider, ILogger logger) + /// SQL connection string used to connect to user database + /// Name of the user table + /// Trigger binding parameter information + /// Provider of unique host identifier + /// Facilitates logging of messages + public SqlTriggerBinding(string connectionString, string tableName, ParameterInfo parameter, IHostIdProvider hostIdProvider, ILogger logger) { - this._tableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); this._connectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString)); + this._tableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); this._parameter = parameter ?? throw new ArgumentNullException(nameof(parameter)); this._hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// - /// Gets the type of the value the Trigger receives from the Executor. + /// Returns the type of trigger value that binds to. /// public Type TriggerValueType => typeof(IReadOnlyList>); - /// - /// Returns an empty binding contract. The type that SqlTriggerAttribute is bound to is checked in - /// - /// public IReadOnlyDictionary BindingDataContract => _emptyBindingContract; - /// - /// Binds the list of represented by "value" with a - /// which (as the name suggests) simply returns "value". - /// - /// The list of data. - /// - /// - /// Unused - /// - /// - /// Thrown if "value" is not of type IReadOnlyList>. - /// - /// - /// The ITriggerData which stores the list of SQL table changes as well as the SimpleValueBinder - /// public Task BindAsync(object value, ValueBindingContext context) { - if (!(value is IReadOnlyList> changes)) - { - throw new InvalidOperationException("The value passed to the SqlTrigger BindAsync must be of type IReadOnlyList>"); - } - - var bindingData = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - { "SqlTrigger", changes } - }; - - return Task.FromResult(new TriggerData(new SimpleValueProvider(this._parameter.ParameterType, changes, this._tableName), bindingData)); + IValueProvider valueProvider = new SqlTriggerValueProvider(this._parameter.ParameterType, value, this._tableName); + return Task.FromResult(new TriggerData(valueProvider, _emptyBindingData)); } - /// - /// Creates a listener that will monitor for changes to the user's table - /// - /// - /// Context for the listener, including the executor that executes the user's function when changes are detected in the user's table - /// - /// - /// Thrown if context is null - /// - /// - /// The listener - /// public async Task CreateListenerAsync(ListenerFactoryContext context) { - if (context == null) - { - throw new ArgumentNullException(nameof(context), "Missing listener context"); - } + _ = context ?? throw new ArgumentNullException(nameof(context), "Missing listener context"); - string userFunctionId = SqlBindingUtilities.AsSingleQuoteEscapedString(await this.GetUserFunctionIdAsync()); + string userFunctionId = await this.GetUserFunctionIdAsync(); return new SqlTriggerListener(this._connectionString, this._tableName, userFunctionId, context.Executor, this._logger); } - /// A description of the SqlTriggerParameter ( public ParameterDescriptor ToParameterDescriptor() { return new SqlTriggerParameterDescriptor { Name = this._parameter.Name, Type = "SqlTrigger", - TableName = _tableName + TableName = this._tableName, }; } /// - /// Creates a unique ID for user function using host ID and method name. + /// Returns an ID that uniquely identifies the user function. /// /// We call the WebJobs SDK library method to generate the host ID. The host ID is essentially a hash of the /// assembly name containing the user function(s). This ensures that if the user ever updates their application, @@ -153,43 +101,7 @@ private async Task GetUserFunctionIdAsync() using (var sha256 = SHA256.Create()) { byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(hostId + functionName)); - - return SqlBindingUtilities.AsSingleQuoteEscapedString(new Guid(hash.Take(16).ToArray()).ToString("N").Substring(0, 16)); - } - } - - /// - /// Simply returns whatever value was passed to it in the constructor without modifying it - /// - internal class SimpleValueProvider : IValueProvider - { - private readonly object _value; - private readonly string _invokeString; - - public SimpleValueProvider(Type type, object value, string invokeString) - { - this.Type = type; - this._value = value; - this._invokeString = invokeString; - } - - /// - /// Returns the type that the trigger binding is bound to (IReadOnlyList{SqlChange{T}}"/>>) - /// - public Type Type { get; } - - public Task GetValueAsync() - { - return Task.FromResult(this._value); - } - - /// - /// Returns the table name that changes are being tracked on - /// - /// - public string ToInvokeString() - { - return this._invokeString; + return new Guid(hash.Take(16).ToArray()).ToString("N").Substring(0, 16); } } } diff --git a/src/TriggerBinding/SqlTriggerBindingProvider.cs b/src/TriggerBinding/SqlTriggerBindingProvider.cs new file mode 100644 index 000000000..e9754a51c --- /dev/null +++ b/src/TriggerBinding/SqlTriggerBindingProvider.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Triggers; +using Microsoft.Azure.WebJobs.Logging; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + /// + /// Provider class for SQL trigger parameter binding. + /// + internal sealed class SqlTriggerBindingProvider : ITriggerBindingProvider + { + private readonly IConfiguration _configuration; + private readonly IHostIdProvider _hostIdProvider; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// Configuration to retrieve settings from + /// Provider of unique host identifier + /// Used to create logger instance + public SqlTriggerBindingProvider(IConfiguration configuration, IHostIdProvider hostIdProvider, ILoggerFactory loggerFactory) + { + this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this._hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); + + _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + this._logger = loggerFactory.CreateLogger(LogCategories.CreateTriggerCategory("Sql")); + } + + /// + /// Creates SQL trigger parameter binding. + /// + /// Contains information about trigger parameter and trigger attributes + /// Thrown if the context is null + /// Thrown if is bound to an invalid parameter type. + /// + /// Null if the user function parameter does not have applied. Otherwise returns an + /// instance, where T is the user-defined POCO type. + /// + public Task TryCreateAsync(TriggerBindingProviderContext context) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + ParameterInfo parameter = context.Parameter; + SqlTriggerAttribute attribute = parameter.GetCustomAttribute(inherit: false); + + if (attribute == null) + { + return Task.FromResult(default(ITriggerBinding)); + } + + if (!IsValidTriggerParameterType(parameter.ParameterType)) + { + throw new InvalidOperationException($"Can't bind SqlTriggerAttribute to type {parameter.ParameterType}." + + " Only IReadOnlyList> is supported, where T is the type of user-defined POCO that" + + " matches the schema of the user table"); + } + + string connectionString = SqlBindingUtilities.GetConnectionString(attribute.ConnectionStringSetting, this._configuration); + + // Extract the POCO type 'T' and use it to instantiate class 'SqlTriggerBinding'. + Type userType = parameter.ParameterType.GetGenericArguments()[0].GetGenericArguments()[0]; + Type bindingType = typeof(SqlTriggerBinding<>).MakeGenericType(userType); + + var constructorParameterTypes = new Type[] { typeof(string), typeof(string), typeof(ParameterInfo), typeof(IHostIdProvider), typeof(ILogger) }; + ConstructorInfo bindingConstructor = bindingType.GetConstructor(constructorParameterTypes); + + object[] constructorParameterValues = new object[] { connectionString, attribute.TableName, parameter, this._hostIdProvider, this._logger }; + var triggerBinding = (ITriggerBinding)bindingConstructor.Invoke(constructorParameterValues); + + return Task.FromResult(triggerBinding); + } + + /// + /// Checks if the type of trigger parameter in the user function is of form . + /// + private static bool IsValidTriggerParameterType(Type type) + { + return + type.IsGenericType && + type.GetGenericTypeDefinition() == typeof(IReadOnlyList<>) && + type.GetGenericArguments()[0].IsGenericType && + type.GetGenericArguments()[0].GetGenericTypeDefinition() == typeof(SqlChange<>); + } + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index b7fbe73c5..9f61fe14e 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -14,9 +14,18 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql { - /// A user-defined POCO that represents a row of the user's table + /// + /// Represents the listener to SQL table changes. + /// + /// POCO class representing the row in the user table internal sealed class SqlTriggerListener : IListener { + private const int ListenerNotStarted = 0; + private const int ListenerStarting = 1; + private const int ListenerStarted = 2; + private const int ListenerStopping = 3; + private const int ListenerStopped = 4; + private readonly SqlObject _userTable; private readonly string _connectionString; private readonly string _userFunctionId; @@ -24,19 +33,16 @@ internal sealed class SqlTriggerListener : IListener private readonly ILogger _logger; private SqlTableChangeMonitor _changeMonitor; - private State _state; + private int _listenerState; /// - /// Initializes a new instance of the > + /// Initializes a new instance of the class. /// - /// The SQL connection string used to connect to the user's database - /// The name of the user table whose changes are being tracked on - /// - /// The unique ID that identifies user function. If multiple application instances are executing the same user - /// function, they are all supposed to have the same user function ID. - /// - /// Used to execute the user's function when changes are detected on "table" - /// Ilogger used to log information and warnings + /// SQL connection string used to connect to user database + /// Name of the user table + /// Unique identifier for the user function + /// Defines contract for triggering user function + /// Facilitates logging of messages public SqlTriggerListener(string connectionString, string tableName, string userFunctionId, ITriggeredFunctionExecutor executor, ILogger logger) { _ = !string.IsNullOrEmpty(connectionString) ? true : throw new ArgumentNullException(nameof(connectionString)); @@ -50,40 +56,39 @@ public SqlTriggerListener(string connectionString, string tableName, string user this._userFunctionId = userFunctionId; this._executor = executor; this._logger = logger; - this._state = State.NotInitialized; + this._listenerState = ListenerNotStarted; } - /// - /// Stops the listener which stops checking for changes on the user's table. - /// public void Cancel() { this.StopAsync(CancellationToken.None).GetAwaiter().GetResult(); } - /// - /// Disposes resources held by the listener to poll for changes. - /// public void Dispose() { - this.StopAsync(CancellationToken.None).GetAwaiter().GetResult(); + // Nothing to dispose. } - /// - /// Starts the listener if it has not yet been started, which starts polling for changes on the user's table. - /// - /// The cancellation token public async Task StartAsync(CancellationToken cancellationToken) { - if (this._state == State.NotInitialized) + int previousState = Interlocked.CompareExchange(ref this._listenerState, ListenerStarting, ListenerNotStarted); + + switch (previousState) + { + case ListenerStarting: throw new InvalidOperationException("The listener is already starting."); + case ListenerStarted: throw new InvalidOperationException("The listener has already started."); + default: break; + } + + try { using (var connection = new SqlConnection(this._connectionString)) { await connection.OpenAsync(cancellationToken); int userTableId = await this.GetUserTableIdAsync(connection, cancellationToken); - IReadOnlyList<(string name, string type)> primaryKeyColumns = await this.GetPrimaryKeyColumnsAsync(connection, cancellationToken); - IReadOnlyList userTableColumns = await this.GetUserTableColumnsAsync(connection, cancellationToken); + IReadOnlyList<(string name, string type)> primaryKeyColumns = await this.GetPrimaryKeyColumnsAsync(connection, userTableId, cancellationToken); + IReadOnlyList userTableColumns = await GetUserTableColumnsAsync(connection, userTableId, cancellationToken); string workerTableName = string.Format(CultureInfo.InvariantCulture, SqlTriggerConstants.WorkerTableNameFormat, $"{this._userFunctionId}_{userTableId}"); @@ -96,7 +101,7 @@ public async Task StartAsync(CancellationToken cancellationToken) transaction.Commit(); } - // TODO: Check if passing cancellation token would be beneficial. + // TODO: Check if passing the cancellation token would be beneficial. this._changeMonitor = new SqlTableChangeMonitor( this._connectionString, userTableId, @@ -108,53 +113,55 @@ public async Task StartAsync(CancellationToken cancellationToken) this._executor, this._logger); - this._state = State.Running; + this._listenerState = ListenerStarted; + this._logger.LogDebug($"Started SQL trigger listener for table: {this._userTable.FullName}, function ID: {this._userFunctionId}."); } } + catch (Exception ex) + { + this._listenerState = ListenerNotStarted; + this._logger.LogError($"Failed to start SQL trigger listener for table: {this._userTable.FullName}, function ID: {this._userFunctionId}. Exception: {ex}"); + + throw; + } } - /// - /// Stops the listener (if it was started), which stops checking for changes on the user's table. - /// public Task StopAsync(CancellationToken cancellationToken) { - // Nothing to stop if the change monitor has either already been stopped or hasn't been started. - if (this._state == State.Running) + int previousState = Interlocked.CompareExchange(ref this._listenerState, ListenerStopping, ListenerStarted); + if (previousState == ListenerStarted) { - this._changeMonitor.Stop(); - this._state = State.Stopped; + this._changeMonitor.Dispose(); + + this._listenerState = ListenerStopped; + this._logger.LogDebug($"Stopped SQL trigger listener for table: {this._userTable.FullName}, function ID: {this._userFunctionId}."); } + return Task.CompletedTask; } /// - /// Returns the OBJECT_ID of userTable + /// Returns the object ID of the user table. /// - /// - /// Thrown if the query to retrieve the OBJECT_ID of the user table fails to correctly execute - /// This can happen if the OBJECT_ID call returns NULL, meaning that the user table might not exist in the database - /// + /// Thrown in case of error when querying the object ID for the user table private async Task GetUserTableIdAsync(SqlConnection connection, CancellationToken cancellationToken) { - string getObjectIdQuery = $"SELECT OBJECT_ID(N{this._userTable.QuotedName}, 'U');"; + string getObjectIdQuery = $"SELECT OBJECT_ID(N{this._userTable.QuotedFullName}, 'U');"; using (var getObjectIdCommand = new SqlCommand(getObjectIdQuery, connection)) { using (SqlDataReader reader = await getObjectIdCommand.ExecuteReaderAsync(cancellationToken)) { - - // TODO: Check if the below if-block ever gets hit. if (!await reader.ReadAsync(cancellationToken)) { - throw new InvalidOperationException($"Failed to determine the OBJECT_ID of the user table {this._userTable.FullName}"); + throw new InvalidOperationException($"Received empty response when querying the object ID for table: {this._userTable.FullName}."); } object userTableId = reader.GetValue(0); if (userTableId is DBNull) { - throw new InvalidOperationException($"Failed to determine the OBJECT_ID of the user table {this._userTable.FullName}. " + - "Possibly the table does not exist in the database."); + throw new InvalidOperationException($"Could not find table: {this._userTable.FullName}."); } return (int)userTableId; @@ -165,11 +172,8 @@ private async Task GetUserTableIdAsync(SqlConnection connection, Cancellati /// /// Gets the names and types of primary key columns of the user's table. /// - /// - /// Thrown if no primary keys are found for the user table. This could be because the user table does not have - /// any primary key columns. - /// - private async Task> GetPrimaryKeyColumnsAsync(SqlConnection connection, CancellationToken cancellationToken) + /// Thrown if there are no primary key columns present in the user table. + private async Task> GetPrimaryKeyColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { string getPrimaryKeyColumnsQuery = $@" SELECT c.name, t.name, c.max_length, c.precision, c.scale @@ -177,7 +181,7 @@ FROM sys.indexes AS i INNER JOIN sys.index_columns AS ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id INNER JOIN sys.columns AS c ON ic.object_id = c.object_id AND ic.column_id = c.column_id INNER JOIN sys.types AS t ON c.user_type_id = t.user_type_id - WHERE i.is_primary_key = 1 AND i.object_id = OBJECT_ID(N{this._userTable.QuotedName}, 'U'); + WHERE i.is_primary_key = 1 AND i.object_id = {userTableId}; "; using (var getPrimaryKeyColumnsCommand = new SqlCommand(getPrimaryKeyColumnsQuery, connection)) @@ -226,13 +230,9 @@ FROM sys.indexes AS i /// /// Gets the column names of the user's table. /// - private async Task> GetUserTableColumnsAsync(SqlConnection connection, CancellationToken cancellationToken) + private static async Task> GetUserTableColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { - string getUserTableColumnsQuery = $@" - SELECT name - FROM sys.columns - WHERE object_id = OBJECT_ID(N{this._userTable.QuotedName}, 'U'); - "; + string getUserTableColumnsQuery = $"SELECT name FROM sys.columns WHERE object_id = {userTableId};"; using (var getUserTableColumnsCommand = new SqlCommand(getUserTableColumnsQuery, connection)) { @@ -336,7 +336,6 @@ private static async Task CreateWorkerTablesAsync( string primaryKeysWithTypes = string.Join(", ", primaryKeyColumns.Select(col => $"{col.name} {col.type}")); string primaryKeys = string.Join(", ", primaryKeyColumns.Select(col => col.name)); - // TODO: Check if some of the table columns below can have 'NOT NULL' property. string createWorkerTableQuery = $@" IF OBJECT_ID(N'{workerTableName}', 'U') IS NULL CREATE TABLE {workerTableName} ( @@ -353,12 +352,5 @@ PRIMARY KEY ({primaryKeys}) await createWorkerTableCommand.ExecuteNonQueryAsync(cancellationToken); } } - - private enum State - { - NotInitialized, - Running, - Stopped, - } } } \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerParameterDescriptor.cs b/src/TriggerBinding/SqlTriggerParameterDescriptor.cs index 20d37160a..f7a4b19f5 100644 --- a/src/TriggerBinding/SqlTriggerParameterDescriptor.cs +++ b/src/TriggerBinding/SqlTriggerParameterDescriptor.cs @@ -7,22 +7,23 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql { + /// + /// Trigger parameter descriptor for . + /// internal sealed class SqlTriggerParameterDescriptor : TriggerParameterDescriptor { /// - /// Name of the table being monitored + /// Name of the user table. /// - public string TableName { get; set; } + public string TableName { private get; set; } /// - /// The reason the user's function was triggered. Specifies the table name that experienced changes - /// as well as the time the changes were detected + /// Returns descriptive reason for why the user function was triggered. /// - /// Unused - /// A string with the reason + /// Collection of function arguments (unused) public override string GetTriggerReason(IDictionary arguments) { - return $"New changes on table {this.TableName} at {DateTime.UtcNow:o}"; + return $"New change detected on table '{this.TableName}' at {DateTime.UtcNow:o}."; } } } \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerValueProvider.cs b/src/TriggerBinding/SqlTriggerValueProvider.cs new file mode 100644 index 000000000..db1d774ac --- /dev/null +++ b/src/TriggerBinding/SqlTriggerValueProvider.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Bindings; + + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + /// + /// Provider for value that will be passed as argument to the triggered function. + /// + internal class SqlTriggerValueProvider : IValueProvider + { + private readonly object _value; + private readonly string _tableName; + + /// + /// Initializes a new instance of the class. + /// + /// Type of the trigger parameter + /// Value of the trigger parameter + /// Name of the user table + public SqlTriggerValueProvider(Type parameterType, object value, string tableName) + { + this.Type = parameterType; + this._value = value; + this._tableName = tableName; + } + + /// + /// Gets the trigger argument value. + /// + public Type Type { get; } + + /// + /// Returns value of the trigger argument. + /// + public Task GetValueAsync() + { + return Task.FromResult(this._value); + } + + public string ToInvokeString() + { + return this._tableName; + } + } +} \ No newline at end of file diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index 3baddfae9..9cb181dff 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -23,7 +23,7 @@ public class IntegrationTestBase : IDisposable /// /// Host process for Azure Function CLI /// - public Process FunctionHost { get; set; } + protected Process FunctionHost { get; private set; } /// /// Host process for Azurite local storage emulator. This is required for non-HTTP trigger functions: @@ -139,24 +139,6 @@ private void ExecuteAllScriptsInFolder(string folder) } } - protected void EnableChangeTracking() - { - string enableChangeTrackingDatabaseQuery = $@"ALTER DATABASE [{this.DatabaseName}] - SET CHANGE_TRACKING = ON - (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON); - "; - this.ExecuteNonQuery(enableChangeTrackingDatabaseQuery); - string enableChangeTrackingTableQuery = $@" - ALTER TABLE [Products] - ENABLE CHANGE_TRACKING - WITH (TRACK_COLUMNS_UPDATED = ON); - ALTER TABLE [ProductsWithMultiplePrimaryColumnsAndIdentity] - ENABLE CHANGE_TRACKING - WITH (TRACK_COLUMNS_UPDATED = ON); - "; - this.ExecuteNonQuery(enableChangeTrackingTableQuery); - } - /// /// This starts the Azurite storage emulator. /// @@ -334,50 +316,49 @@ private static string GetPathToBin() public void Dispose() { + // Try to clean up after test run, but don't consider it a failure if we can't for some reason try { this.Connection.Close(); + this.Connection.Dispose(); } catch (Exception e1) { this.TestOutput.WriteLine($"Failed to close connection. Error: {e1.Message}"); } - finally + + try { - this.Connection.Dispose(); + this.FunctionHost?.Kill(); + this.FunctionHost?.Dispose(); + } + catch (Exception e2) + { + this.TestOutput.WriteLine($"Failed to stop function host, Error: {e2.Message}"); + } - // Try to clean up after test run, but don't consider it a failure if we can't for some reason - try - { - this.FunctionHost?.Kill(); - this.FunctionHost?.Dispose(); - } - catch (Exception e2) - { - this.TestOutput.WriteLine($"Failed to stop function host, Error: {e2.Message}"); - } + try + { + this.AzuriteHost?.Kill(); + this.AzuriteHost?.Dispose(); + } + catch (Exception e3) + { + this.TestOutput.WriteLine($"Failed to stop Azurite, Error: {e3.Message}"); + } - try - { - this.AzuriteHost?.Kill(); - this.AzuriteHost?.Dispose(); - } - catch (Exception e3) - { - this.TestOutput.WriteLine($"Failed to stop Azurite, Error: {e3.Message}"); - } - try - { - // Drop the test database - using var masterConnection = new SqlConnection(this.MasterConnectionString); - masterConnection.Open(); - TestUtils.ExecuteNonQuery(masterConnection, $"DROP DATABASE IF EXISTS {this.DatabaseName}"); - } - catch (Exception e4) - { - this.TestOutput.WriteLine($"Failed to drop {this.DatabaseName}, Error: {e4.Message}"); - } + try + { + // Drop the test database + using var masterConnection = new SqlConnection(this.MasterConnectionString); + masterConnection.Open(); + TestUtils.ExecuteNonQuery(masterConnection, $"DROP DATABASE IF EXISTS {this.DatabaseName}"); + } + catch (Exception e4) + { + this.TestOutput.WriteLine($"Failed to drop {this.DatabaseName}, Error: {e4.Message}"); } + GC.SuppressFinalize(this); } } diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 429d8edc9..d637a7ac4 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -1,14 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System.Diagnostics; -using System.Text; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; +using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; -using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; -using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration { @@ -17,159 +18,151 @@ public class SqlTriggerBindingIntegrationTests : IntegrationTestBase { public SqlTriggerBindingIntegrationTests(ITestOutputHelper output) : base(output) { + this.EnableChangeTrackingForDatabase(); } - /// - /// Tests for insertion of products triggering the function. - /// [Fact] - public async void InsertProductsTest() + public async void BasicTriggerTest() { - int countInsert = 0; - this.EnableChangeTracking(); + this.EnableChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); - this.FunctionHost.OutputDataReceived += (object sender, DataReceivedEventArgs e) => - { - if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to Products table row")) - { - countInsert++; - } - }; - - Product[] products = GetProducts(3, 100); - this.InsertProducts(products); - await Task.Delay(5000); - - Assert.Equal(3, countInsert); + var changes = new List>(); + this.MonitorProductChanges(changes); + + // Considering the polling interval of 5 seconds and batch-size of 10, it should take around 15 seconds to + // process 30 insert operations. Similar reasoning is used to set delays for update and delete operations. + this.InsertProducts(1, 30); + await Task.Delay(TimeSpan.FromSeconds(16)); + ValidateProductChanges(changes, 1, 30, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); + changes.Clear(); + + // All table columns (not just the columns that were updated) would be returned for update operation. + this.UpdateProducts(1, 20); + await Task.Delay(TimeSpan.FromSeconds(11)); + ValidateProductChanges(changes, 1, 20, SqlChangeOperation.Update, id => $"Updated Product {id}", id => id * 100); + changes.Clear(); + + // The properties corresponding to non-primary key columns would be set to the C# type's default values + // (null and 0) for delete operation. + this.DeleteProducts(11, 30); + await Task.Delay(TimeSpan.FromSeconds(11)); + ValidateProductChanges(changes, 11, 30, SqlChangeOperation.Delete, _ => null, _ => 0); + changes.Clear(); } - /// - /// Tests insertion into table with multiple primary key columns. - /// + [Fact] - public void InsertMultiplePrimaryKeyColumnsTest() + public async void MultiOperationTriggerTest() { - this.EnableChangeTracking(); - this.StartFunctionHost(nameof(ProductsWithMultiplePrimaryColumnsTrigger), Common.SupportedLanguages.CSharp); - var taskCompletionSource = new TaskCompletionSource(); - this.FunctionHost.OutputDataReceived += (object sender, DataReceivedEventArgs e) => - { - if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to ProductsWithMultiplePrimaryColumns table row")) - { - taskCompletionSource.SetResult(true); - } - }; + this.EnableChangeTrackingForTable("Products"); + this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); - string query = $@"INSERT INTO dbo.ProductsWithMultiplePrimaryColumnsAndIdentity VALUES(123, 'ProductTest', 100);"; - this.ExecuteNonQuery(query); - taskCompletionSource.Task.Wait(10000); + var changes = new List>(); + this.MonitorProductChanges(changes); + + // Insert + multiple updates to a row are treated as single insert with latest row values. + this.InsertProducts(1, 5); + this.UpdateProducts(1, 5); + this.UpdateProducts(1, 5); + await Task.Delay(TimeSpan.FromSeconds(6)); + ValidateProductChanges(changes, 1, 5, SqlChangeOperation.Insert, id => $"Updated Updated Product {id}", id => id * 100); + changes.Clear(); + + // Multiple updates to a row are treated as single update with latest row values. + this.InsertProducts(6, 10); + await Task.Delay(TimeSpan.FromSeconds(6)); + changes.Clear(); + this.UpdateProducts(6, 10); + this.UpdateProducts(6, 10); + await Task.Delay(TimeSpan.FromSeconds(6)); + ValidateProductChanges(changes, 6, 10, SqlChangeOperation.Update, id => $"Updated Updated Product {id}", id => id * 100); + changes.Clear(); + + // Insert + (zero or more updates) + delete to a row are treated as single delete with default values for non-primary columns. + this.InsertProducts(11, 20); + this.UpdateProducts(11, 20); + this.DeleteProducts(11, 20); + await Task.Delay(TimeSpan.FromSeconds(6)); + ValidateProductChanges(changes, 11, 20, SqlChangeOperation.Delete, _ => null, _ => 0); + changes.Clear(); + } - Assert.True(taskCompletionSource.Task.Result); + private void EnableChangeTrackingForDatabase() + { + this.ExecuteNonQuery($@" + ALTER DATABASE [{this.DatabaseName}] + SET CHANGE_TRACKING = ON + (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON); + "); + } + private void EnableChangeTrackingForTable(string tableName) + { + this.ExecuteNonQuery($@" + ALTER TABLE [dbo].[{tableName}] + ENABLE CHANGE_TRACKING + WITH (TRACK_COLUMNS_UPDATED = OFF); + "); } - /// - /// Tests for behaviour of the trigger when insertion, updates, and deletes occur. - /// - [Fact] - public async void InsertUpdateDeleteProductsTest() + private void MonitorProductChanges(List> changes) { - int countInsert = 0; - int countUpdate = 0; - int countDelete = 0; - this.EnableChangeTracking(); - this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); - this.FunctionHost.OutputDataReceived += (object sender, DataReceivedEventArgs e) => + int index = 0; + string prefix = "SQL Changes: "; + + this.FunctionHost.OutputDataReceived += (sender, e) => { - if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to Products table row: Insert")) + if (e.Data != null && (index = e.Data.IndexOf(prefix, StringComparison.Ordinal)) >= 0) { - countInsert++; - } - if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to Products table row: Update")) - { - countUpdate++; - } - if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"Change occurred to Products table row: Delete")) - { - countDelete++; + string json = e.Data[(index + prefix.Length)..]; + changes.AddRange(JsonConvert.DeserializeObject>>(json)); } }; - Product[] products = GetProducts(3, 100); - this.InsertProducts(products); - await Task.Delay(500); - this.UpdateProducts(products.Take(2).ToArray()); - await Task.Delay(500); - this.DeleteProducts(products.Take(1).ToArray()); - - await Task.Delay(5000); - - //Since insert and update counts as a single insert and insert and delete counts as a single delete - Assert.Equal(2, countInsert); - Assert.Equal(0, countUpdate); - Assert.Equal(1, countDelete); - } - private static Product[] GetProducts(int n, int cost) + private void InsertProducts(int first_id, int last_id) { - var result = new Product[n]; - for (int i = 1; i <= n; i++) - { - result[i - 1] = new Product - { - ProductID = i, - Name = "test", - Cost = cost * i - }; - } - return result; + int count = last_id - first_id + 1; + this.ExecuteNonQuery( + "INSERT INTO [dbo].[Products] VALUES\n" + + string.Join(",\n", Enumerable.Range(first_id, count).Select(id => $"({id}, 'Product {id}', {id * 100})")) + ";"); } - private void InsertProducts(Product[] products) - { - if (products.Length == 0) - { - return; - } - - var queryBuilder = new StringBuilder(); - foreach (Product p in products) - { - queryBuilder.AppendLine($"INSERT INTO dbo.Products VALUES({p.ProductID}, '{p.Name}', {p.Cost});"); - } - this.ExecuteNonQuery(queryBuilder.ToString()); - } - private void UpdateProducts(Product[] products) + private void UpdateProducts(int first_id, int last_id) { - if (products.Length == 0) - { - return; - } - - var queryBuilder = new StringBuilder(); - foreach (Product p in products) - { - string newName = p.Name + "Update"; - queryBuilder.AppendLine($"UPDATE dbo.Products set Name = '{newName}' where ProductId = {p.ProductID};"); - } + int count = last_id - first_id + 1; + this.ExecuteNonQuery( + "UPDATE [dbo].[Products]\n" + + "SET Name = 'Updated ' + Name\n" + + "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(first_id, count)) + ");"); + } - this.ExecuteNonQuery(queryBuilder.ToString()); + private void DeleteProducts(int first_id, int last_id) + { + int count = last_id - first_id + 1; + this.ExecuteNonQuery( + "DELETE FROM [dbo].[Products]\n" + + "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(first_id, count)) + ");"); } - private void DeleteProducts(Product[] products) + + private static void ValidateProductChanges(List> changes, int first_id, int last_id, + SqlChangeOperation operation, Func getName, Func getCost) { - if (products.Length == 0) - { - return; - } + int count = last_id - first_id + 1; + Assert.Equal(count, changes.Count); - var queryBuilder = new StringBuilder(); - foreach (Product p in products) + int id = first_id; + foreach (SqlChange change in changes) { - queryBuilder.AppendLine($"DELETE from dbo.Products where ProductId = {p.ProductID};"); + Assert.Equal(operation, change.Operation); + Product product = change.Item; + Assert.NotNull(product); + Assert.Equal(id, product.ProductID); + Assert.Equal(getName(id), product.Name); + Assert.Equal(getCost(id), product.Cost); + id += 1; } - - this.ExecuteNonQuery(queryBuilder.ToString()); } } } \ No newline at end of file diff --git a/test/Unit/SqlTriggerBindingTests.cs b/test/Unit/SqlTriggerBindingTests.cs index 348076286..a874d34a8 100644 --- a/test/Unit/SqlTriggerBindingTests.cs +++ b/test/Unit/SqlTriggerBindingTests.cs @@ -2,105 +2,85 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; -using System.Reflection; using System.Collections.Generic; +using System.Reflection; using System.Threading; -using Microsoft.Extensions.Configuration; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Triggers; -using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Xunit; using Moq; +using Xunit; namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Unit { - public class TriggerBindingTests + public class SqlTriggerBindingTests { - private static readonly Mock config = new Mock(); - private static readonly Mock hostIdProvider = new Mock(); - private static readonly Mock loggerFactory = new Mock(); - private static readonly Mock mockExecutor = new Mock(); - private static readonly Mock logger = new Mock(); - private static readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - [Fact] - public void TestTriggerBindingProviderNullConfig() + public async Task SqlTriggerBindingProvider_ReturnsNullBindingForParameterWithoutAttribute() { - Assert.Throws(() => new SqlTriggerAttributeBindingProvider(null, hostIdProvider.Object, loggerFactory.Object)); - Assert.Throws(() => new SqlTriggerAttributeBindingProvider(config.Object, null, loggerFactory.Object)); - Assert.Throws(() => new SqlTriggerAttributeBindingProvider(config.Object, hostIdProvider.Object, null)); + Type parameterType = typeof(IReadOnlyList>); + ITriggerBinding binding = await CreateTriggerBindingAsync(parameterType, nameof(UserFunctionWithoutAttribute)); + Assert.Null(binding); } [Fact] - public async void TestTriggerAttributeBindingProviderNullContext() + public async Task SqlTriggerBindingProvider_ThrowsForMissingConnectionString() { - var configProvider = new SqlTriggerAttributeBindingProvider(config.Object, hostIdProvider.Object, loggerFactory.Object); - await Assert.ThrowsAsync(() => configProvider.TryCreateAsync(null)); + Type parameterType = typeof(IReadOnlyList>); + Task testCode() { return CreateTriggerBindingAsync(parameterType, nameof(UserFunctionWithoutConnectionString)); } + ArgumentException exception = await Assert.ThrowsAsync(testCode); + + Assert.Equal( + "Must specify ConnectionStringSetting, which should refer to the name of an app setting that contains a SQL connection string", + exception.Message); } - [Fact] - public void TestTriggerListenerNullConfig() + [Theory] + [InlineData(typeof(object))] + [InlineData(typeof(SqlChange))] + [InlineData(typeof(IEnumerable>))] + [InlineData(typeof(IReadOnlyList))] + [InlineData(typeof(IReadOnlyList>))] + public async Task SqlTriggerBindingProvider_ThrowsForInvalidTriggerParameterType(Type parameterType) { - string connectionString = "testConnectionString"; - string tableName = "testTableName"; - string userFunctionId = "testUserFunctionId"; + Task testCode() { return CreateTriggerBindingAsync(parameterType, nameof(UserFunctionWithoutConnectionString)); } + InvalidOperationException exception = await Assert.ThrowsAsync(testCode); - Assert.Throws(() => new SqlTriggerListener(null, tableName, userFunctionId, mockExecutor.Object, logger.Object)); - Assert.Throws(() => new SqlTriggerListener(connectionString, null, userFunctionId, mockExecutor.Object, logger.Object)); - Assert.Throws(() => new SqlTriggerListener(connectionString, tableName, null, mockExecutor.Object, logger.Object)); - Assert.Throws(() => new SqlTriggerListener(connectionString, tableName, userFunctionId, null, logger.Object)); - Assert.Throws(() => new SqlTriggerListener(connectionString, tableName, userFunctionId, mockExecutor.Object, null)); + Assert.Equal( + $"Can't bind SqlTriggerAttribute to type {parameterType}. Only IReadOnlyList> is supported, where T is the type of user-defined POCO that matches the schema of the user table", + exception.Message); } [Fact] - public void TestTriggerBindingNullConfig() + public async Task SqlTriggerBindingProvider_ReturnsBindingForValidTriggerParameterType() { - string connectionString = "testConnectionString"; - string tableName = "testTableName"; - - Assert.Throws(() => new SqlTriggerBinding(null, connectionString, TriggerBindingFunctionTest.GetParamForChanges(), hostIdProvider.Object, logger.Object)); - Assert.Throws(() => new SqlTriggerBinding(tableName, null, TriggerBindingFunctionTest.GetParamForChanges(), hostIdProvider.Object, logger.Object)); - Assert.Throws(() => new SqlTriggerBinding(tableName, connectionString, null, hostIdProvider.Object, logger.Object)); - Assert.Throws(() => new SqlTriggerBinding(tableName, connectionString, TriggerBindingFunctionTest.GetParamForChanges(), null, logger.Object)); - Assert.Throws(() => new SqlTriggerBinding(tableName, connectionString, TriggerBindingFunctionTest.GetParamForChanges(), hostIdProvider.Object, null)); + Type parameterType = typeof(IReadOnlyList>); + ITriggerBinding binding = await CreateTriggerBindingAsync(parameterType, nameof(UserFunctionWithAttribute)); + Assert.NotNull(binding); } - [Fact] - public async void TestTriggerBindingProviderWithInvalidParameter() + private static async Task CreateTriggerBindingAsync(Type parameterType, string methodName) { - var triggerBindingProviderContext = new TriggerBindingProviderContext(TriggerBindingFunctionTest.GetParamForChanges(), cancellationTokenSource.Token); - var triggerAttributeBindingProvider = new SqlTriggerAttributeBindingProvider(config.Object, hostIdProvider.Object, loggerFactory.Object); + var provider = new SqlTriggerBindingProvider( + Mock.Of(c => c["dummyConnectionStringSetting"] == "dummyConnectionString"), + Mock.Of(), + Mock.Of(f => f.CreateLogger(It.IsAny()) == Mock.Of())); - //Trying to create a SqlTriggerBinding with IEnumerable> type for the changes - //This is expected to throw an exception as the type expected for receiving the changes is IReadOnlyList> - await Assert.ThrowsAsync(() => triggerAttributeBindingProvider.TryCreateAsync(triggerBindingProviderContext)); - } + // Possibly the simplest way to construct a ParameterInfo object. + ParameterInfo parameter = typeof(SqlTriggerBindingTests) + .GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Static) + .MakeGenericMethod(parameterType) + .GetParameters()[0]; - /// - /// Creating a function using trigger with wrong parameter for changes field. - /// - private static class TriggerBindingFunctionTest - { - /// - ///Example function created with wrong parameter - /// - public static void InvalidParameterType( - [SqlTrigger("[dbo].[Employees]", ConnectionStringSetting = "SqlConnectionString")] - IEnumerable> changes, - ILogger logger) - { - logger.LogInformation(changes.ToString()); - } - /// - ///Gets the parameter info for changes in the function - /// - public static ParameterInfo GetParamForChanges() - { - MethodInfo methodInfo = typeof(TriggerBindingFunctionTest).GetMethod("InvalidParameterType", BindingFlags.Public | BindingFlags.Static); - ParameterInfo[] parameters = methodInfo.GetParameters(); - return parameters[^2]; - } + return await provider.TryCreateAsync(new TriggerBindingProviderContext(parameter, CancellationToken.None)); } + + private static void UserFunctionWithoutAttribute(T _) { } + + private static void UserFunctionWithoutConnectionString([SqlTrigger("dummyTableName")] T _) { } + + private static void UserFunctionWithAttribute([SqlTrigger("dummyTableName", ConnectionStringSetting = "dummyConnectionStringSetting")] T _) { } } } \ No newline at end of file From 443692feb26597b1aaed5e15a77ad3270cdc5a9a Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:49:45 +0530 Subject: [PATCH 03/77] Handle all supported characters in table and column names --- src/TriggerBinding/SqlTableChangeMonitor.cs | 39 +++++++++++++++------ src/TriggerBinding/SqlTriggerListener.cs | 4 +-- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 1215e9a44..d7c7685a4 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -6,6 +6,7 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -41,6 +42,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly string _workerTableName; private readonly IReadOnlyList _userTableColumns; private readonly IReadOnlyList _primaryKeyColumns; + private readonly IReadOnlyList<(string col, string hash)> _primaryKeyColumnHashes; private readonly IReadOnlyList _rowMatchConditions; private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; @@ -94,10 +96,11 @@ public SqlTableChangeMonitor( this._workerTableName = workerTableName; this._userTableColumns = primaryKeyColumns.Concat(userTableColumns.Except(primaryKeyColumns)).ToList(); this._primaryKeyColumns = primaryKeyColumns; + this._primaryKeyColumnHashes = primaryKeyColumns.Select(col => (col, GetColumnHash(col))).ToList(); // Prep search-conditions that will be used besides WHERE clause to match table rows. this._rowMatchConditions = Enumerable.Range(0, BatchSize) - .Select(index => string.Join(" AND ", primaryKeyColumns.Select(col => $"{col} = @{col}_{index}"))) + .Select(index => string.Join(" AND ", this._primaryKeyColumnHashes.Select(ch => $"{ch.col.AsBracketQuotedString()} = @{ch.hash}_{index}"))) .ToList(); this._executor = executor; @@ -540,6 +543,20 @@ private static SqlChangeOperation GetChangeOperation(IReadOnlyDictionary + /// Creates GUID string from column name that can be used as name of SQL local variable. The column names + /// cannot be used directly as variable names as they may contain characters like ',', '.', '+', spaces, etc. + /// that are not allowed in variable names. + /// + private static string GetColumnHash(string columnName) + { + using (var sha256 = SHA256.Create()) + { + byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(columnName)); + return new Guid(hash.Take(16).ToArray()).ToString("N"); + } + } + /// /// Builds the command to update the global state table in the case of a new minimum valid version number. /// Sets the LastSyncVersion for this _userTable to be the new minimum valid version number. @@ -575,9 +592,9 @@ IF @last_sync_version < @min_valid_version /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransaction transaction) { - string selectList = string.Join(", ", this._userTableColumns.Select(col => this._primaryKeyColumns.Contains(col) ? $"c.{col}" : $"u.{col}")); - string userTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col} = u.{col}")); - string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col} = w.{col}")); + string selectList = string.Join(", ", this._userTableColumns.Select(col => this._primaryKeyColumns.Contains(col) ? $"c.{col.AsBracketQuotedString()}" : $"u.{col.AsBracketQuotedString()}")); + string userTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = u.{col.AsBracketQuotedString()}")); + string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = w.{col.AsBracketQuotedString()}")); string getChangesQuery = $@" DECLARE @last_sync_version bigint; @@ -589,9 +606,9 @@ SELECT TOP {BatchSize} {selectList}, c.SYS_CHANGE_VERSION, c.SYS_CHANGE_OPERATION, w.ChangeVersion, w.AttemptCount, w.LeaseExpirationTime - FROM CHANGETABLE (CHANGES {this._userTable.FullName}, @last_sync_version) AS c + FROM CHANGETABLE (CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} - LEFT OUTER JOIN {this._userTable.FullName} AS u ON {userTableJoinCondition} + LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCondition} WHERE (w.LeaseExpirationTime IS NULL AND (w.ChangeVersion IS NULL OR w.ChangeVersion < c.SYS_CHANGE_VERSION) OR w.LeaseExpirationTime < SYSDATETIME()) AND @@ -615,7 +632,7 @@ private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransa for (int index = 0; index < this._rows.Count; index++) { - string valuesList = string.Join(", ", this._primaryKeyColumns.Select(col => $"@{col}_{index}")); + string valuesList = string.Join(", ", this._primaryKeyColumnHashes.Select(ch => $"@{ch.hash}_{index}")); string changeVersion = this._rows[index]["SYS_CHANGE_VERSION"]; acquireLeasesQuery.Append($@" @@ -697,7 +714,7 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, SqlTransaction transaction, long newLastSyncVersion) { - string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col} = w.{col}")); + string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = w.{col.AsBracketQuotedString()}")); // TODO: Need to think through all cases to ensure the query below is correct, especially with use of < vs <=. string updateTablesPostInvocationQuery = $@" @@ -709,7 +726,7 @@ private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, Sql DECLARE @unprocessed_changes bigint; SELECT @unprocessed_changes = COUNT(*) FROM ( SELECT c.SYS_CHANGE_VERSION - FROM CHANGETABLE(CHANGES {this._userTable.FullName}, @current_last_sync_version) AS c + FROM CHANGETABLE (CHANGES {this._userTable.BracketQuotedFullName}, @current_last_sync_version) AS c LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} WHERE c.SYS_CHANGE_VERSION <= {newLastSyncVersion} AND @@ -751,9 +768,9 @@ private SqlCommand GetSqlCommandWithParameters(string commandText, SqlConnection for (int index = 0; index < this._rows.Count; index++) { - foreach (string col in this._primaryKeyColumns) + foreach ((string col, string hash) in this._primaryKeyColumnHashes) { - command.Parameters.Add(new SqlParameter($"@{col}_{index}", this._rows[index][col])); + command.Parameters.Add(new SqlParameter($"@{hash}_{index}", this._rows[index][col])); } } diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 9f61fe14e..6de03f910 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -333,8 +333,8 @@ private static async Task CreateWorkerTablesAsync( IReadOnlyList<(string name, string type)> primaryKeyColumns, CancellationToken cancellationToken) { - string primaryKeysWithTypes = string.Join(", ", primaryKeyColumns.Select(col => $"{col.name} {col.type}")); - string primaryKeys = string.Join(", ", primaryKeyColumns.Select(col => col.name)); + string primaryKeysWithTypes = string.Join(", ", primaryKeyColumns.Select(col => $"{col.name.AsBracketQuotedString()} [{col.type}]")); + string primaryKeys = string.Join(", ", primaryKeyColumns.Select(col => col.name.AsBracketQuotedString())); string createWorkerTableQuery = $@" IF OBJECT_ID(N'{workerTableName}', 'U') IS NULL From 2a14a73a2e35d6caee6fdfbeb4b4fa85577d9bb1 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Thu, 28 Jul 2022 22:02:40 +0530 Subject: [PATCH 04/77] Address review comments on Readme file --- README.md | 18 +++++++++--------- src/TriggerBinding/SqlTriggerListener.cs | 4 ++-- .../SqlTriggerBindingIntegrationTests.cs | 3 +-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index db980867a..b9b9a6db8 100644 --- a/README.md +++ b/README.md @@ -95,11 +95,10 @@ ALTER TABLE ['{table_name}'] ADD CONSTRAINT PKey PRIMARY KEY CLUSTERED  ```sql ALTER DATABASE ['your database name'] SET CHANGE_TRACKING = ON -(CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON) +(CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON); ALTER TABLE ['your table name'] -ENABLE CHANGE_TRACKING -WITH (TRACK_COLUMNS_UPDATED = ON) +ENABLE CHANGE_TRACKING; ``` @@ -458,6 +457,8 @@ public static async Task Run( The output binding takes a list of rows to be upserted into a user table. If the primary key value of the row already exists in the table, the row is interpreted as an update, meaning that the values of the other columns in the table for that primary key are updated. If the primary key value does not exist in the table, the row is interpreted as an insert. The upserting of the rows is batched by the output binding code. + > **NOTE:** By default the Output binding uses the T-SQL [MERGE](https://docs.microsoft.com/sql/t-sql/statements/merge-transact-sql) statement which requires [SELECT](https://docs.microsoft.com/sql/t-sql/statements/merge-transact-sql#permissions) permissions on the target database. + The output binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/SqlAttribute.cs): - **CommandText**: Passed as a constructor argument to the binding. Represents the name of the table into which rows will be upserted. @@ -581,6 +582,8 @@ This changes if one of the primary key columns is an identity column though. In ### Trigger Binding +> **NOTE:** Trigger binding support is only available for C# functions at present. + #### Change Tracking The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sql/relational-databases/track-changes/about-change-tracking-sql-server) functionality to monitor the user table for changes. As such, it is necessary to enable change tracking on the SQL database and the SQL table before using the trigger support. The change tracking can be enabled through the following two queries. @@ -590,7 +593,7 @@ The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sq ```sql ALTER DATABASE ['your database name'] SET CHANGE_TRACKING = ON - (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON) + (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON); ``` The `CHANGE_RETENTION` option specifies the duration for which the changes are retained in the change tracking table. This may affect the trigger functionality. For example, if the user application is turned off for several days and then resumed, it will only be able to catch the changes that occurred in past two days with the above query. Hence, please update the value of `CHANGE_RETENTION` to suit your requirements. The `AUTO_CLEANUP` option is used to enable or disable the clean-up task that removes the stale data. Please refer to SQL Server documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-database) for more information. @@ -599,13 +602,10 @@ The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sq ```sql ALTER TABLE dbo.Employees - ENABLE CHANGE_TRACKING - WITH (TRACK_COLUMNS_UPDATED = ON) + ENABLE CHANGE_TRACKING; ``` - The `TRACK_COLUMNS_UPDATED` option lets the SQL server to store information about which table columns were updated. At present, the trigger binding does not use of this information, though that functionality can be added in future. For more information, please refer to the documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-table). - - The trigger needs to have read access on the table being monitored for changes as well as to the change tracking system tables. It also needs write access to an `az_func` schema within the database, where it will create additional worker tables to store the trigger states and leases. Each function trigger will thus have an associated change tracking table and worker table. + For more information, please refer to the documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-table). The trigger needs to have read access on the table being monitored for changes as well as to the change tracking system tables. It also needs write access to an `az_func` schema within the database, where it will create additional worker tables to store the trigger states and leases. Each function trigger will thus have an associated change tracking table and worker table. #### Trigger Samples The trigger binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/TriggerBinding/SqlTriggerAttribute.cs) diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 6de03f910..9601205bc 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -97,7 +97,7 @@ public async Task StartAsync(CancellationToken cancellationToken) await CreateSchemaAsync(connection, transaction, cancellationToken); await CreateGlobalStateTableAsync(connection, transaction, cancellationToken); await this.InsertGlobalStateTableRowAsync(connection, transaction, userTableId, cancellationToken); - await CreateWorkerTablesAsync(connection, transaction, workerTableName, primaryKeyColumns, cancellationToken); + await CreateWorkerTableAsync(connection, transaction, workerTableName, primaryKeyColumns, cancellationToken); transaction.Commit(); } @@ -326,7 +326,7 @@ INSERT INTO {SqlTriggerConstants.GlobalStateTableName} /// /// Creates the worker table associated with the user's table, if one does not already exist. /// - private static async Task CreateWorkerTablesAsync( + private static async Task CreateWorkerTableAsync( SqlConnection connection, SqlTransaction transaction, string workerTableName, diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index d637a7ac4..1c66354d2 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -101,8 +101,7 @@ private void EnableChangeTrackingForTable(string tableName) { this.ExecuteNonQuery($@" ALTER TABLE [dbo].[{tableName}] - ENABLE CHANGE_TRACKING - WITH (TRACK_COLUMNS_UPDATED = OFF); + ENABLE CHANGE_TRACKING; "); } From b011dea87b81d96cc0f97ca481f6b3aac15099cc Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Fri, 29 Jul 2022 14:42:34 +0530 Subject: [PATCH 05/77] Address review comments on SqlTriggerListener --- src/TriggerBinding/SqlTriggerListener.cs | 159 ++++++++++++----------- 1 file changed, 82 insertions(+), 77 deletions(-) diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 9601205bc..2e45ff1ab 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -114,13 +114,13 @@ public async Task StartAsync(CancellationToken cancellationToken) this._logger); this._listenerState = ListenerStarted; - this._logger.LogDebug($"Started SQL trigger listener for table: {this._userTable.FullName}, function ID: {this._userFunctionId}."); + this._logger.LogDebug($"Started SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); } } catch (Exception ex) { this._listenerState = ListenerNotStarted; - this._logger.LogError($"Failed to start SQL trigger listener for table: {this._userTable.FullName}, function ID: {this._userFunctionId}. Exception: {ex}"); + this._logger.LogError($"Failed to start SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'. Exception: {ex}"); throw; } @@ -134,7 +134,7 @@ public Task StopAsync(CancellationToken cancellationToken) this._changeMonitor.Dispose(); this._listenerState = ListenerStopped; - this._logger.LogDebug($"Stopped SQL trigger listener for table: {this._userTable.FullName}, function ID: {this._userFunctionId}."); + this._logger.LogDebug($"Stopped SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); } return Task.CompletedTask; @@ -149,30 +149,30 @@ private async Task GetUserTableIdAsync(SqlConnection connection, Cancellati string getObjectIdQuery = $"SELECT OBJECT_ID(N{this._userTable.QuotedFullName}, 'U');"; using (var getObjectIdCommand = new SqlCommand(getObjectIdQuery, connection)) + using (SqlDataReader reader = await getObjectIdCommand.ExecuteReaderAsync(cancellationToken)) { - using (SqlDataReader reader = await getObjectIdCommand.ExecuteReaderAsync(cancellationToken)) + if (!await reader.ReadAsync(cancellationToken)) { - if (!await reader.ReadAsync(cancellationToken)) - { - throw new InvalidOperationException($"Received empty response when querying the object ID for table: {this._userTable.FullName}."); - } - - object userTableId = reader.GetValue(0); + throw new InvalidOperationException($"Received empty response when querying the object ID for table: '{this._userTable.FullName}'."); + } - if (userTableId is DBNull) - { - throw new InvalidOperationException($"Could not find table: {this._userTable.FullName}."); - } + object userTableId = reader.GetValue(0); - return (int)userTableId; + if (userTableId is DBNull) + { + throw new InvalidOperationException($"Could not find table: '{this._userTable.FullName}'."); } + + return (int)userTableId; } } /// - /// Gets the names and types of primary key columns of the user's table. + /// Gets the names and types of primary key columns of the user table. /// - /// Thrown if there are no primary key columns present in the user table. + /// + /// Thrown if there are no primary key columns present in the user table or if their names conflict with columns in worker table. + /// private async Task> GetPrimaryKeyColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { string getPrimaryKeyColumnsQuery = $@" @@ -185,74 +185,74 @@ FROM sys.indexes AS i "; using (var getPrimaryKeyColumnsCommand = new SqlCommand(getPrimaryKeyColumnsQuery, connection)) + using (SqlDataReader reader = await getPrimaryKeyColumnsCommand.ExecuteReaderAsync(cancellationToken)) { - using (SqlDataReader reader = await getPrimaryKeyColumnsCommand.ExecuteReaderAsync(cancellationToken)) - { + string[] reservedColumnNames = new string[] { "ChangeVersion", "AttemptCount", "LeaseExpirationTime" }; + string[] variableLengthTypes = new string[] { "varchar", "nvarchar", "nchar", "char", "binary", "varbinary" }; + string[] variablePrecisionTypes = new string[] { "numeric", "decimal" }; - string[] variableLengthTypes = new string[] { "varchar", "nvarchar", "nchar", "char", "binary", "varbinary" }; - string[] variablePrecisionTypes = new string[] { "numeric", "decimal" }; + var primaryKeyColumns = new List<(string name, string type)>(); - var primaryKeyColumns = new List<(string name, string type)>(); + while (await reader.ReadAsync(cancellationToken)) + { + string name = reader.GetString(0); - while (await reader.ReadAsync(cancellationToken)) + if (reservedColumnNames.Contains(name)) { - string type = reader.GetString(1); - - if (variableLengthTypes.Contains(type)) - { - // Special "max" case. I'm actually not sure it's valid to have varchar(max) as a primary key because - // it exceeds the byte limit of an index field (900 bytes), but just in case - short length = reader.GetInt16(2); - type += length == -1 ? "(max)" : $"({length})"; - } - else if (variablePrecisionTypes.Contains(type)) - { - byte precision = reader.GetByte(3); - byte scale = reader.GetByte(4); - type += $"({precision},{scale})"; - } - - primaryKeyColumns.Add((name: reader.GetString(0), type)); + throw new InvalidOperationException($"Found reserved column name: '{name}' in table: '{this._userTable.FullName}'."); } - if (primaryKeyColumns.Count == 0) + string type = reader.GetString(1); + + if (variableLengthTypes.Contains(type)) { - throw new InvalidOperationException($"Unable to determine the primary keys of user table {this._userTable.FullName}. " + - "Potentially, the table does not have any primary key columns. A primary key is required for every " + - "user table for which changes are being monitored."); + // Special "max" case. I'm actually not sure it's valid to have varchar(max) as a primary key because + // it exceeds the byte limit of an index field (900 bytes), but just in case + short length = reader.GetInt16(2); + type += length == -1 ? "(max)" : $"({length})"; + } + else if (variablePrecisionTypes.Contains(type)) + { + byte precision = reader.GetByte(3); + byte scale = reader.GetByte(4); + type += $"({precision},{scale})"; } - return primaryKeyColumns; + primaryKeyColumns.Add((name: reader.GetString(0), type)); + } + + if (primaryKeyColumns.Count == 0) + { + throw new InvalidOperationException($"Could not find primary key created in table: '{this._userTable.FullName}'."); } + + return primaryKeyColumns; } } /// - /// Gets the column names of the user's table. + /// Gets the column names of the user table. /// private static async Task> GetUserTableColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { string getUserTableColumnsQuery = $"SELECT name FROM sys.columns WHERE object_id = {userTableId};"; using (var getUserTableColumnsCommand = new SqlCommand(getUserTableColumnsQuery, connection)) + using (SqlDataReader reader = await getUserTableColumnsCommand.ExecuteReaderAsync(cancellationToken)) { - using (SqlDataReader reader = await getUserTableColumnsCommand.ExecuteReaderAsync(cancellationToken)) - { - - var userTableColumns = new List(); + var userTableColumns = new List(); - while (await reader.ReadAsync(cancellationToken)) - { - userTableColumns.Add(reader.GetString(0)); - } - - return userTableColumns; + while (await reader.ReadAsync(cancellationToken)) + { + userTableColumns.Add(reader.GetString(0)); } + + return userTableColumns; } } /// - /// Creates the schema where the worker tables will be located if it does not already exist. + /// Creates the schema for global state table and worker tables, if it does not already exist. /// private static async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { @@ -289,42 +289,47 @@ PRIMARY KEY (UserFunctionID, UserTableID) } /// - /// Inserts row for the user table and the function inside the global state table, if one does not already exist. + /// Inserts row for the 'user function and table' inside the global state table, if one does not already exist. /// private async Task InsertGlobalStateTableRowAsync(SqlConnection connection, SqlTransaction transaction, int userTableId, CancellationToken cancellationToken) { + object minValidVersion; + + string getMinValidVersionQuery = $"SELECT CHANGE_TRACKING_MIN_VALID_VERSION({userTableId});"; + + using (var getMinValidVersionCommand = new SqlCommand(getMinValidVersionQuery, connection, transaction)) + using (SqlDataReader reader = await getMinValidVersionCommand.ExecuteReaderAsync(cancellationToken)) + { + if (!await reader.ReadAsync(cancellationToken)) + { + throw new InvalidOperationException($"Received empty response when querying the 'change tracking min valid version' for table: '{this._userTable.FullName}'."); + } + + minValidVersion = reader.GetValue(0); + + if (minValidVersion is DBNull) + { + throw new InvalidOperationException($"Could not find change tracking enabled for table: '{this._userTable.FullName}'."); + } + } + string insertRowGlobalStateTableQuery = $@" IF NOT EXISTS ( SELECT * FROM {SqlTriggerConstants.GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {userTableId} ) INSERT INTO {SqlTriggerConstants.GlobalStateTableName} - VALUES ('{this._userFunctionId}', {userTableId}, CHANGE_TRACKING_MIN_VALID_VERSION({userTableId})); + VALUES ('{this._userFunctionId}', {userTableId}, {(long)minValidVersion}); "; using (var insertRowGlobalStateTableCommand = new SqlCommand(insertRowGlobalStateTableQuery, connection, transaction)) { - try - { - await insertRowGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); - } - catch (Exception e) - { - // Could fail if we try to insert a NULL value into the LastSyncVersion, which happens when - // CHANGE_TRACKING_MIN_VALID_VERSION returns NULL for the user table, meaning that change tracking is - // not enabled for either the database or table (or both). - - string errorMessage = $"Failed to start processing changes to table {this._userTable.FullName}, " + - $"potentially because change tracking was not enabled for the table or database {connection.Database}."; - - this._logger.LogWarning(errorMessage + $" Exact exception thrown is {e.Message}"); - throw new InvalidOperationException(errorMessage); - } + await insertRowGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); } } /// - /// Creates the worker table associated with the user's table, if one does not already exist. + /// Creates the worker table for the 'user function and table', if one does not already exist. /// private static async Task CreateWorkerTableAsync( SqlConnection connection, @@ -333,7 +338,7 @@ private static async Task CreateWorkerTableAsync( IReadOnlyList<(string name, string type)> primaryKeyColumns, CancellationToken cancellationToken) { - string primaryKeysWithTypes = string.Join(", ", primaryKeyColumns.Select(col => $"{col.name.AsBracketQuotedString()} [{col.type}]")); + string primaryKeysWithTypes = string.Join(",\n", primaryKeyColumns.Select(col => $"{col.name.AsBracketQuotedString()} [{col.type}]")); string primaryKeys = string.Join(", ", primaryKeyColumns.Select(col => col.name.AsBracketQuotedString())); string createWorkerTableQuery = $@" From 0497251276d55253907c468b304ea98cff2dc85d Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Fri, 29 Jul 2022 23:23:18 +0530 Subject: [PATCH 06/77] Address review comments on SqlTableChangeMonitor --- README.md | 2 + src/TriggerBinding/SqlTableChangeMonitor.cs | 115 ++++++++---------- src/TriggerBinding/SqlTriggerListener.cs | 15 ++- .../SqlTriggerBindingIntegrationTests.cs | 6 +- 4 files changed, 66 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index b9b9a6db8..f8a5db1de 100644 --- a/README.md +++ b/README.md @@ -641,6 +641,8 @@ public static void Run( - Output bindings against tables with columns of data types `NTEXT`, `TEXT`, or `IMAGE` are not supported and data upserts will fail. These types [will be removed](https://docs.microsoft.com/sql/t-sql/data-types/ntext-text-and-image-transact-sql) in a future version of SQL Server and are not compatible with the `OPENJSON` function used by this Azure Functions binding. +- Trigger bindings will exhibit undefined behavior if the SQL table schema gets modified while the user application is running, for example, if a column is added, renamed or deleted or if the primary key is modified or deleted. In such cases, restarting the application should help resolve any errors. + ## Telemetry This extension collect usage data in order to help us improve your experience. The data is anonymous and doesn't include any personal information. You can opt-out of telemetry by setting the `AZUREFUNCTIONS_SQLBINDINGS_TELEMETRY_OPTOUT` environment variable or the `AzureFunctionsSqlBindingsTelemetryOptOut` app setting (in your `*.settings.json` file) to '1', 'true' or 'yes'; diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index d7c7685a4..bf1d16b46 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -18,22 +18,21 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql { /// - /// Periodically polls SQL's change table to determine if any new changes have occurred to a user's table. + /// Watches for changes in the user table, invokes user function if changes are found, and manages leases. /// - /// - /// Note that there is no possiblity of SQL injection in the raw queries we generate. All parameters that involve - /// inserting data from a user table are sanitized. All other parameters are generated exclusively using information - /// about the user table's schema (such as primary key column names), data stored in SQL's internal change table, or - /// data stored in our own worker table. - /// - /// A user-defined POCO that represents a row of the user's table + /// POCO class representing the row in the user table internal sealed class SqlTableChangeMonitor : IDisposable { public const int BatchSize = 10; - public const int MaxAttemptCount = 5; - public const int MaxLeaseRenewalCount = 5; - public const int LeaseIntervalInSeconds = 30; public const int PollingIntervalInSeconds = 5; + public const int MaxAttemptCount = 5; + + // Leases are held for approximately (LeaseRenewalIntervalInSeconds * MaxLeaseRenewalCount) seconds. It is + // required to have at least one of (LeaseRenewalIntervalInSeconds / LeaseRenewalIntervalInSeconds) attempts to + // renew the lease succeed to prevent it from expiring. + public const int MaxLeaseRenewalCount = 10; + public const int LeaseIntervalInSeconds = 60; + public const int LeaseRenewalIntervalInSeconds = 15; private readonly string _connectionString; private readonly int _userTableId; @@ -51,8 +50,9 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly CancellationTokenSource _cancellationTokenSourceRenewLeases; private CancellationTokenSource _cancellationTokenSourceExecutor; - // It should be impossible for multiple threads to access these at the same time because of the semaphore we use. + // The semaphore ensures that mutable class members such as this._rows are accessed by only one thread at a time. private readonly SemaphoreSlim _rowsLock; + private IReadOnlyList> _rows; private int _leaseRenewalCount; private State _state = State.CheckingForChanges; @@ -60,15 +60,15 @@ internal sealed class SqlTableChangeMonitor : IDisposable /// /// Initializes a new instance of the > class. /// - /// The SQL connection string used to connect to the user's database - /// The OBJECT_ID of the user table whose changes are being tracked on - /// The name of the user table - /// The unique ID that identifies user function - /// The name of the worker table + /// SQL connection string used to connect to user database + /// SQL object ID of the user table + /// instance created with user table name + /// Unique identifier for the user function + /// Name of the worker table /// List of all column names in the user table /// List of primary key column names in the user table - /// Used to execute the user's function when changes are detected on "table" - /// Ilogger used to log information and warnings + /// Defines contract for triggering user function + /// Facilitates logging of messages public SqlTableChangeMonitor( string connectionString, int userTableId, @@ -94,7 +94,7 @@ public SqlTableChangeMonitor( this._userTable = userTable; this._userFunctionId = userFunctionId; this._workerTableName = workerTableName; - this._userTableColumns = primaryKeyColumns.Concat(userTableColumns.Except(primaryKeyColumns)).ToList(); + this._userTableColumns = userTableColumns; this._primaryKeyColumns = primaryKeyColumns; this._primaryKeyColumnHashes = primaryKeyColumns.Select(col => (col, GetColumnHash(col))).ToList(); @@ -124,11 +124,6 @@ public SqlTableChangeMonitor( #pragma warning restore CS4014 } - /// - /// Stops the change monitor which stops polling for changes on the user's table. If the change monitor is - /// currently executing a set of changes, it is only stopped once execution is finished and the user's function - /// is triggered (whether or not the trigger is successful). - /// public void Dispose() { this._cancellationTokenSourceCheckForChanges.Cancel(); @@ -152,11 +147,11 @@ private async Task RunChangeConsumptionLoopAsync() { await connection.OpenAsync(token); + // Check for cancellation request only after a cycle of checking and processing of changes completes. while (!token.IsCancellationRequested) { if (this._state == State.CheckingForChanges) { - // What should we do if this call gets stuck? await this.GetChangesAsync(token); await this.ProcessChangesAsync(token); } @@ -233,16 +228,16 @@ private async Task GetChangesAsync(CancellationToken token) } catch (Exception ex) { - this._logger.LogError("Commit Exception Type: {0}", ex.GetType()); - this._logger.LogError(" Message: {0}", ex.Message); + this._logger.LogError($"Failed to query list of changes for table '{this._userTable.FullName}' due to exception: {ex.GetType()}." + + $" Exception message: {ex.Message}"); + try { transaction.Rollback(); } catch (Exception ex2) { - this._logger.LogError("Rollback Exception Type: {0}", ex2.GetType()); - this._logger.LogError(" Message: {0}", ex2.Message); + this._logger.LogError($"Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}"); } } } @@ -253,7 +248,8 @@ private async Task GetChangesAsync(CancellationToken token) // If there's an exception in any part of the process, we want to clear all of our data in memory and // retry checking for changes again. this._rows = new List>(); - this._logger.LogWarning($"Failed to check {this._userTable.FullName} for new changes due to error: {e.Message}"); + this._logger.LogError($"Failed to check for changes in table '{this._userTable.FullName}' due to exception: {e.GetType()}." + + $" Exception message: {e.Message}"); } } @@ -274,16 +270,16 @@ private async Task ProcessChangesAsync(CancellationToken token) } catch (Exception e) { - await this.ClearRowsAsync( - $"Failed to extract user table data from table {this._userTable.FullName} associated " + - $"with change metadata due to error: {e.Message}", true); + this._logger.LogError($"Failed to compose trigger parameter value for table: '{this._userTable.FullName} due to exception: {e.GetType()}." + + $" Exception message: {e.Message}"); + + await this.ClearRowsAsync(true); } if (changes != null) { - FunctionResult result = await this._executor.TryExecuteAsync( - new TriggeredFunctionData() { TriggerValue = changes }, - this._cancellationTokenSourceExecutor.Token); + var input = new TriggeredFunctionData() { TriggerValue = changes }; + FunctionResult result = await this._executor.TryExecuteAsync(input, this._cancellationTokenSourceExecutor.Token); if (result.Succeeded) { @@ -293,9 +289,10 @@ await this.ClearRowsAsync( { // In the future might make sense to retry executing the function, but for now we just let // another worker try. - await this.ClearRowsAsync( - $"Failed to trigger user's function for table {this._userTable.FullName} due to " + - $"error: {result.Exception.Message}", true); + this._logger.LogError($"Failed to trigger user function for table: '{this._userTable.FullName} due to exception: {result.Exception.GetType()}." + + $" Exception message: {result.Exception.Message}"); + + await this.ClearRowsAsync(true); } } } @@ -318,11 +315,8 @@ private async void RunLeaseRenewalLoopAsync() while (!token.IsCancellationRequested) { await this._rowsLock.WaitAsync(token); - await this.RenewLeasesAsync(connection, token); - - // Want to make sure to renew the leases before they expire, so we renew them twice per lease period. - await Task.Delay(TimeSpan.FromSeconds(LeaseIntervalInSeconds / 2), token); + await Task.Delay(TimeSpan.FromSeconds(LeaseRenewalIntervalInSeconds), token); } } } @@ -362,7 +356,7 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken // (see https://docs.microsoft.com/dotnet/csharp/language-reference/keywords/try-finally, third // paragraph). If we fail to renew the leases, multiple workers could be processing the same change // data, but we have functionality in place to deal with this (see design doc). - this._logger.LogError($"Failed to renew leases due to error: {e.Message}"); + this._logger.LogError($"Failed to renew leases due to exception: {e.GetType()}. Exception message: {e.Message}"); } finally { @@ -395,15 +389,11 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken /// /// Resets the in-memory state of the change monitor and sets it to start polling for changes again. /// - /// - /// The error messages the logger will report describing the reason function execution failed (used only in the case of a failure). - /// /// True if ClearRowsAsync should acquire the "_rowsLock" (only true in the case of a failure) - private async Task ClearRowsAsync(string error, bool acquireLock) + private async Task ClearRowsAsync(bool acquireLock) { if (acquireLock) { - this._logger.LogError(error); await this._rowsLock.WaitAsync(); } @@ -449,16 +439,16 @@ private async Task ReleaseLeasesAsync(CancellationToken token) } catch (Exception ex) { - this._logger.LogError("Commit Exception Type: {0}", ex.GetType()); - this._logger.LogError(" Message: {0}", ex.Message); + this._logger.LogError($"Failed to execute SQL commands to release leases for table '{this._userTable.FullName}' due to exception: {ex.GetType()}." + + $" Exception message: {ex.Message}"); + try { transaction.Rollback(); } catch (Exception ex2) { - this._logger.LogError("Rollback Exception Type: {0}", ex2.GetType()); - this._logger.LogError(" Message: {0}", ex2.Message); + this._logger.LogError($"Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}"); } } } @@ -470,21 +460,19 @@ private async Task ReleaseLeasesAsync(CancellationToken token) // What should we do if releasing the leases fails? We could try to release them again or just wait, // since eventually the lease time will expire. Then another thread will re-process the same changes // though, so less than ideal. But for now that's the functionality. - this._logger.LogError($"Failed to release leases for user table {this._userTable.FullName} due to error: {e.Message}"); + this._logger.LogError($"Failed to release leases for table '{this._userTable.FullName}' due to exception: {e.GetType()}." + + $" Exception message: {e.Message}"); } finally { // Want to do this before releasing the lock in case the renew leases thread wakes up. It will see that // the state is checking for changes and not renew the (just released) leases. - await this.ClearRowsAsync(string.Empty, false); + await this.ClearRowsAsync(false); } } /// - /// Calculates the new version number to attempt to update LastSyncVersion in global state table to. If all - /// version numbers in _rows are the same, use that version number. If they aren't, use the second largest - /// version number. For an explanation as to why this method was chosen, see 9c in Steps of Operation in this - /// design doc: https://microsoft-my.sharepoint.com/:w:/p/t-sotevo/EQdANWq9ZWpKm8e48TdzUwcBGZW07vJmLf8TL_rtEG8ixQ?e=owN2EX. + /// Computes the version number that can be potentially used as the new LastSyncVersion in the global state table. /// private long RecomputeLastSyncVersion() { @@ -495,7 +483,7 @@ private long RecomputeLastSyncVersion() changeVersionSet.Add(long.Parse(changeVersion, CultureInfo.InvariantCulture)); } - // If there are at least two version numbers in this set, return the second highest one. Otherwise, return + // If there are more than one version numbers in the set, return the second highest one. Otherwise, return // the only version number in the set. return changeVersionSet.ElementAt(changeVersionSet.Count > 1 ? changeVersionSet.Count - 2 : 0); } @@ -606,7 +594,7 @@ SELECT TOP {BatchSize} {selectList}, c.SYS_CHANGE_VERSION, c.SYS_CHANGE_OPERATION, w.ChangeVersion, w.AttemptCount, w.LeaseExpirationTime - FROM CHANGETABLE (CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c + FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCondition} WHERE @@ -716,7 +704,6 @@ private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, Sql { string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = w.{col.AsBracketQuotedString()}")); - // TODO: Need to think through all cases to ensure the query below is correct, especially with use of < vs <=. string updateTablesPostInvocationQuery = $@" DECLARE @current_last_sync_version bigint; SELECT @current_last_sync_version = LastSyncVersion @@ -726,7 +713,7 @@ private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, Sql DECLARE @unprocessed_changes bigint; SELECT @unprocessed_changes = COUNT(*) FROM ( SELECT c.SYS_CHANGE_VERSION - FROM CHANGETABLE (CHANGES {this._userTable.BracketQuotedFullName}, @current_last_sync_version) AS c + FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_sync_version) AS c LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} WHERE c.SYS_CHANGE_VERSION <= {newLastSyncVersion} AND diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 2e45ff1ab..fd8f82f34 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -88,9 +88,10 @@ public async Task StartAsync(CancellationToken cancellationToken) int userTableId = await this.GetUserTableIdAsync(connection, cancellationToken); IReadOnlyList<(string name, string type)> primaryKeyColumns = await this.GetPrimaryKeyColumnsAsync(connection, userTableId, cancellationToken); - IReadOnlyList userTableColumns = await GetUserTableColumnsAsync(connection, userTableId, cancellationToken); + IReadOnlyList userTableColumns = await this.GetUserTableColumnsAsync(connection, userTableId, cancellationToken); string workerTableName = string.Format(CultureInfo.InvariantCulture, SqlTriggerConstants.WorkerTableNameFormat, $"{this._userFunctionId}_{userTableId}"); + this._logger.LogDebug($"Worker table name: '{workerTableName}'."); using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) { @@ -101,6 +102,8 @@ public async Task StartAsync(CancellationToken cancellationToken) transaction.Commit(); } + this._logger.LogInformation($"Starting SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); + // TODO: Check if passing the cancellation token would be beneficial. this._changeMonitor = new SqlTableChangeMonitor( this._connectionString, @@ -114,7 +117,7 @@ public async Task StartAsync(CancellationToken cancellationToken) this._logger); this._listenerState = ListenerStarted; - this._logger.LogDebug($"Started SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); + this._logger.LogInformation($"Started SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); } } catch (Exception ex) @@ -134,7 +137,7 @@ public Task StopAsync(CancellationToken cancellationToken) this._changeMonitor.Dispose(); this._listenerState = ListenerStopped; - this._logger.LogDebug($"Stopped SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); + this._logger.LogInformation($"Stopped SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); } return Task.CompletedTask; @@ -204,7 +207,7 @@ FROM sys.indexes AS i string type = reader.GetString(1); - if (variableLengthTypes.Contains(type)) + if (variableLengthTypes.Contains(type, StringComparer.OrdinalIgnoreCase)) { // Special "max" case. I'm actually not sure it's valid to have varchar(max) as a primary key because // it exceeds the byte limit of an index field (900 bytes), but just in case @@ -226,6 +229,7 @@ FROM sys.indexes AS i throw new InvalidOperationException($"Could not find primary key created in table: '{this._userTable.FullName}'."); } + this._logger.LogDebug($"Primary key column names(types): {string.Join(", ", primaryKeyColumns.Select(col => $"'{col.name}({col.type})'"))}."); return primaryKeyColumns; } } @@ -233,7 +237,7 @@ FROM sys.indexes AS i /// /// Gets the column names of the user table. /// - private static async Task> GetUserTableColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) + private async Task> GetUserTableColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { string getUserTableColumnsQuery = $"SELECT name FROM sys.columns WHERE object_id = {userTableId};"; @@ -247,6 +251,7 @@ private static async Task> GetUserTableColumnsAsync(SqlCon userTableColumns.Add(reader.GetString(0)); } + this._logger.LogDebug($"User table column names: {string.Join(", ", userTableColumns.Select(col => $"'{col}'"))}."); return userTableColumns; } } diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 1c66354d2..e060642a3 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -33,20 +33,20 @@ public async void BasicTriggerTest() // Considering the polling interval of 5 seconds and batch-size of 10, it should take around 15 seconds to // process 30 insert operations. Similar reasoning is used to set delays for update and delete operations. this.InsertProducts(1, 30); - await Task.Delay(TimeSpan.FromSeconds(16)); + await Task.Delay(TimeSpan.FromSeconds(20)); ValidateProductChanges(changes, 1, 30, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); changes.Clear(); // All table columns (not just the columns that were updated) would be returned for update operation. this.UpdateProducts(1, 20); - await Task.Delay(TimeSpan.FromSeconds(11)); + await Task.Delay(TimeSpan.FromSeconds(15)); ValidateProductChanges(changes, 1, 20, SqlChangeOperation.Update, id => $"Updated Product {id}", id => id * 100); changes.Clear(); // The properties corresponding to non-primary key columns would be set to the C# type's default values // (null and 0) for delete operation. this.DeleteProducts(11, 30); - await Task.Delay(TimeSpan.FromSeconds(11)); + await Task.Delay(TimeSpan.FromSeconds(15)); ValidateProductChanges(changes, 11, 30, SqlChangeOperation.Delete, _ => null, _ => 0); changes.Clear(); } From 24886a156bfc7244d4bcffd3d5e50e973eba5d85 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Sat, 30 Jul 2022 00:54:09 +0530 Subject: [PATCH 07/77] Fix code comment --- src/TriggerBinding/SqlTableChangeMonitor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index bf1d16b46..671408df3 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -28,7 +28,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable public const int MaxAttemptCount = 5; // Leases are held for approximately (LeaseRenewalIntervalInSeconds * MaxLeaseRenewalCount) seconds. It is - // required to have at least one of (LeaseRenewalIntervalInSeconds / LeaseRenewalIntervalInSeconds) attempts to + // required to have at least one of (LeaseIntervalInSeconds / LeaseRenewalIntervalInSeconds) attempts to // renew the lease succeed to prevent it from expiring. public const int MaxLeaseRenewalCount = 10; public const int LeaseIntervalInSeconds = 60; From db8fdc99b1508b4717b856cd9d5629e6560d9038 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Tue, 9 Aug 2022 07:11:43 +0530 Subject: [PATCH 08/77] Add telemetry events for trigger binding (#289) * Add telemetry for trigger binding * Remove table name from telemetry properties * Open SQL connection to fetch server version * Set connection props after connection is opened --- src/Telemetry/Telemetry.cs | 42 ++++++- src/TriggerBinding/SqlTableChangeMonitor.cs | 115 +++++++++++++++++--- src/TriggerBinding/SqlTriggerListener.cs | 65 +++++++++-- 3 files changed, 192 insertions(+), 30 deletions(-) diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index 3d4008abd..3b7a6d01b 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -320,18 +320,32 @@ public enum ConvertType /// public enum TelemetryEventName { + AcquireLeaseEnd, + AcquireLeaseStart, AddAsync, - Create, Convert, + Create, Error, FlushAsync, GetCaseSensitivity, + GetChangesEnd, + GetChangesStart, GetColumnDefinitions, GetPrimaryKeys, GetTableInfoEnd, GetTableInfoStart, + ReleaseLeasesEnd, + ReleaseLeasesStart, + RenewLeasesEnd, + RenewLeasesStart, + StartListenerEnd, + StartListenerStart, + StopListenerEnd, + StopListenerStart, TableInfoCacheHit, TableInfoCacheMiss, + TriggerFunctionEnd, + TriggerFunctionStart, UpsertEnd, UpsertStart, } @@ -341,13 +355,15 @@ public enum TelemetryEventName /// public enum TelemetryPropertyName { + ErrorCode, ErrorName, ExceptionType, HasIdentityColumn, QueryType, ServerVersion, Type, - ErrorCode + UserFunctionId, + WorkerTableName, } /// @@ -355,13 +371,22 @@ public enum TelemetryPropertyName /// public enum TelemetryMeasureName { + AcquireLeasesDurationMs, BatchCount, CommandDurationMs, + CreatedSchemaDurationMs, + CreateGlobalStateTableDurationMs, + CreateWorkerTableDurationMs, DurationMs, GetCaseSensitivityDurationMs, + GetChangesDurationMs, GetColumnDefinitionsDurationMs, GetPrimaryKeysDurationMs, - TransactionDurationMs + InsertGlobalStateTableRowDurationMs, + ReleaseLeasesDurationMs, + SetLastSyncVersionDurationMs, + TransactionDurationMs, + UpdateLastSyncVersionDurationMs, } /// @@ -369,15 +394,24 @@ public enum TelemetryMeasureName /// public enum TelemetryErrorName { + ConsumeChangesLoop, Convert, FlushAsync, GetCaseSensitivity, + GetChanges, + GetChangesRollback, GetColumnDefinitions, GetColumnDefinitionsTableDoesNotExist, GetPrimaryKeys, - NoPrimaryKeys, MissingPrimaryKeys, + NoPrimaryKeys, + ProcessChanges, PropsNotExistOnTable, + ReleaseLeases, + ReleaseLeasesRollback, + RenewLeases, + RenewLeasesLoop, + StartListener, Upsert, UpsertRollback, } diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 671408df3..40c0add6a 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; @@ -10,6 +11,8 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry; +using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry; using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; @@ -53,6 +56,8 @@ internal sealed class SqlTableChangeMonitor : IDisposable // The semaphore ensures that mutable class members such as this._rows are accessed by only one thread at a time. private readonly SemaphoreSlim _rowsLock; + private readonly IDictionary _telemetryProps; + private IReadOnlyList> _rows; private int _leaseRenewalCount; private State _state = State.CheckingForChanges; @@ -69,6 +74,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable /// List of primary key column names in the user table /// Defines contract for triggering user function /// Facilitates logging of messages + /// Properties passed in telemetry events public SqlTableChangeMonitor( string connectionString, int userTableId, @@ -78,7 +84,8 @@ public SqlTableChangeMonitor( IReadOnlyList userTableColumns, IReadOnlyList primaryKeyColumns, ITriggeredFunctionExecutor executor, - ILogger logger) + ILogger logger, + IDictionary telemetryProps) { _ = !string.IsNullOrEmpty(connectionString) ? true : throw new ArgumentNullException(nameof(connectionString)); _ = !string.IsNullOrEmpty(userTable.FullName) ? true : throw new ArgumentNullException(nameof(userTable)); @@ -110,6 +117,8 @@ public SqlTableChangeMonitor( this._cancellationTokenSourceRenewLeases = new CancellationTokenSource(); this._cancellationTokenSourceExecutor = new CancellationTokenSource(); + this._telemetryProps = telemetryProps; + this._rowsLock = new SemaphoreSlim(1); this._rows = new List>(); this._leaseRenewalCount = 0; @@ -139,6 +148,8 @@ public void Dispose() /// private async Task RunChangeConsumptionLoopAsync() { + this._logger.LogInformation("Starting change consumption loop."); + try { CancellationToken token = this._cancellationTokenSourceCheckForChanges.Token; @@ -166,7 +177,8 @@ private async Task RunChangeConsumptionLoopAsync() // throws an exception if it's cancelled. if (e.GetType() != typeof(TaskCanceledException)) { - this._logger.LogError(e.Message); + this._logger.LogError($"Exiting change consumption loop due to exception: {e.GetType()}. Exception message: {e.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ConsumeChangesLoop, e, this._telemetryProps); } } finally @@ -185,12 +197,17 @@ private async Task RunChangeConsumptionLoopAsync() /// private async Task GetChangesAsync(CancellationToken token) { + TelemetryInstance.TrackEvent(TelemetryEventName.GetChangesStart, this._telemetryProps); + try { using (var connection = new SqlConnection(this._connectionString)) { await connection.OpenAsync(token); + var transactionSw = Stopwatch.StartNew(); + long setLastSyncVersionDurationMs = 0L, getChangesDurationMs = 0L, acquireLeasesDurationMs = 0L; + using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) { try @@ -198,13 +215,17 @@ private async Task GetChangesAsync(CancellationToken token) // Update the version number stored in the global state table if necessary before using it. using (SqlCommand updateTablesPreInvocationCommand = this.BuildUpdateTablesPreInvocation(connection, transaction)) { + var commandSw = Stopwatch.StartNew(); await updateTablesPreInvocationCommand.ExecuteNonQueryAsync(token); + setLastSyncVersionDurationMs = commandSw.ElapsedMilliseconds; } // Use the version number to query for new changes. using (SqlCommand getChangesCommand = this.BuildGetChangesCommand(connection, transaction)) { + var commandSw = Stopwatch.StartNew(); var rows = new List>(); + using (SqlDataReader reader = await getChangesCommand.ExecuteReaderAsync(token)) { while (await reader.ReadAsync(token)) @@ -214,22 +235,39 @@ private async Task GetChangesAsync(CancellationToken token) } this._rows = rows; + getChangesDurationMs = commandSw.ElapsedMilliseconds; } + this._logger.LogDebug($"Changed rows count: {this._rows.Count}."); + // If changes were found, acquire leases on them. if (this._rows.Count > 0) { using (SqlCommand acquireLeasesCommand = this.BuildAcquireLeasesCommand(connection, transaction)) { + var commandSw = Stopwatch.StartNew(); await acquireLeasesCommand.ExecuteNonQueryAsync(token); + acquireLeasesDurationMs = commandSw.ElapsedMilliseconds; } } + transaction.Commit(); + + var measures = new Dictionary + { + [TelemetryMeasureName.SetLastSyncVersionDurationMs.ToString()] = setLastSyncVersionDurationMs, + [TelemetryMeasureName.GetChangesDurationMs.ToString()] = getChangesDurationMs, + [TelemetryMeasureName.AcquireLeasesDurationMs.ToString()] = acquireLeasesDurationMs, + [TelemetryMeasureName.TransactionDurationMs.ToString()] = transactionSw.ElapsedMilliseconds, + [TelemetryMeasureName.BatchCount.ToString()] = this._rows.Count, + }; + + TelemetryInstance.TrackEvent(TelemetryEventName.GetChangesEnd, this._telemetryProps, measures); } catch (Exception ex) { - this._logger.LogError($"Failed to query list of changes for table '{this._userTable.FullName}' due to exception: {ex.GetType()}." + - $" Exception message: {ex.Message}"); + this._logger.LogError($"Failed to query list of changes for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.GetChanges, ex, this._telemetryProps); try { @@ -238,6 +276,7 @@ private async Task GetChangesAsync(CancellationToken token) catch (Exception ex2) { this._logger.LogError($"Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.GetChangesRollback, ex2, this._telemetryProps); } } } @@ -248,8 +287,8 @@ private async Task GetChangesAsync(CancellationToken token) // If there's an exception in any part of the process, we want to clear all of our data in memory and // retry checking for changes again. this._rows = new List>(); - this._logger.LogError($"Failed to check for changes in table '{this._userTable.FullName}' due to exception: {e.GetType()}." + - $" Exception message: {e.Message}"); + this._logger.LogError($"Failed to check for changes in table '{this._userTable.FullName}' due to exception: {e.GetType()}. Exception message: {e.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.GetChanges, e, this._telemetryProps); } } @@ -270,27 +309,37 @@ private async Task ProcessChangesAsync(CancellationToken token) } catch (Exception e) { - this._logger.LogError($"Failed to compose trigger parameter value for table: '{this._userTable.FullName} due to exception: {e.GetType()}." + - $" Exception message: {e.Message}"); - + this._logger.LogError($"Failed to compose trigger parameter value for table: '{this._userTable.FullName} due to exception: {e.GetType()}. Exception message: {e.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ProcessChanges, e, this._telemetryProps); await this.ClearRowsAsync(true); } if (changes != null) { var input = new TriggeredFunctionData() { TriggerValue = changes }; + + TelemetryInstance.TrackEvent(TelemetryEventName.TriggerFunctionStart, this._telemetryProps); + var stopwatch = Stopwatch.StartNew(); + FunctionResult result = await this._executor.TryExecuteAsync(input, this._cancellationTokenSourceExecutor.Token); + var measures = new Dictionary + { + [TelemetryMeasureName.DurationMs.ToString()] = stopwatch.ElapsedMilliseconds, + [TelemetryMeasureName.BatchCount.ToString()] = this._rows.Count, + }; + if (result.Succeeded) { + TelemetryInstance.TrackEvent(TelemetryEventName.TriggerFunctionEnd, this._telemetryProps, measures); await this.ReleaseLeasesAsync(token); } else { // In the future might make sense to retry executing the function, but for now we just let // another worker try. - this._logger.LogError($"Failed to trigger user function for table: '{this._userTable.FullName} due to exception: {result.Exception.GetType()}." + - $" Exception message: {result.Exception.Message}"); + this._logger.LogError($"Failed to trigger user function for table: '{this._userTable.FullName} due to exception: {result.Exception.GetType()}. Exception message: {result.Exception.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ProcessChanges, result.Exception, this._telemetryProps, measures); await this.ClearRowsAsync(true); } @@ -304,6 +353,8 @@ private async Task ProcessChangesAsync(CancellationToken token) /// private async void RunLeaseRenewalLoopAsync() { + this._logger.LogInformation("Starting lease renewal loop."); + try { CancellationToken token = this._cancellationTokenSourceRenewLeases.Token; @@ -326,7 +377,8 @@ private async void RunLeaseRenewalLoopAsync() // an exception if it's cancelled. if (e.GetType() != typeof(TaskCanceledException)) { - this._logger.LogError(e.Message); + this._logger.LogError($"Exiting lease renewal loop due to exception: {e.GetType()}. Exception message: {e.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.RenewLeasesLoop, e); } } finally @@ -346,7 +398,17 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken // deleted by a cleanup task, it shouldn't renew the lease on it anyways. using (SqlCommand renewLeasesCommand = this.BuildRenewLeasesCommand(connection)) { + TelemetryInstance.TrackEvent(TelemetryEventName.RenewLeasesStart, this._telemetryProps); + var stopwatch = Stopwatch.StartNew(); + await renewLeasesCommand.ExecuteNonQueryAsync(token); + + var measures = new Dictionary + { + [TelemetryMeasureName.DurationMs.ToString()] = stopwatch.ElapsedMilliseconds, + }; + + TelemetryInstance.TrackEvent(TelemetryEventName.RenewLeasesEnd, this._telemetryProps, measures); } } } @@ -357,6 +419,7 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken // paragraph). If we fail to renew the leases, multiple workers could be processing the same change // data, but we have functionality in place to deal with this (see design doc). this._logger.LogError($"Failed to renew leases due to exception: {e.GetType()}. Exception message: {e.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.RenewLeases, e, this._telemetryProps); } finally { @@ -409,6 +472,8 @@ private async Task ClearRowsAsync(bool acquireLock) /// private async Task ReleaseLeasesAsync(CancellationToken token) { + TelemetryInstance.TrackEvent(TelemetryEventName.ReleaseLeasesStart, this._telemetryProps); + // Don't want to change the "_rows" while another thread is attempting to renew leases on them. await this._rowsLock.WaitAsync(token); long newLastSyncVersion = this.RecomputeLastSyncVersion(); @@ -418,6 +483,10 @@ private async Task ReleaseLeasesAsync(CancellationToken token) using (var connection = new SqlConnection(this._connectionString)) { await connection.OpenAsync(token); + + var transactionSw = Stopwatch.StartNew(); + long releaseLeasesDurationMs = 0L, updateLastSyncVersionDurationMs = 0L; + using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) { try @@ -425,22 +494,34 @@ private async Task ReleaseLeasesAsync(CancellationToken token) // Release the leases held on "_rows". using (SqlCommand releaseLeasesCommand = this.BuildReleaseLeasesCommand(connection, transaction)) { + var commandSw = Stopwatch.StartNew(); await releaseLeasesCommand.ExecuteNonQueryAsync(token); + releaseLeasesDurationMs = commandSw.ElapsedMilliseconds; } // Update the global state table if we have processed all changes with ChangeVersion <= newLastSyncVersion, // and clean up the worker table to remove all rows with ChangeVersion <= newLastSyncVersion. using (SqlCommand updateTablesPostInvocationCommand = this.BuildUpdateTablesPostInvocation(connection, transaction, newLastSyncVersion)) { + var commandSw = Stopwatch.StartNew(); await updateTablesPostInvocationCommand.ExecuteNonQueryAsync(token); + updateLastSyncVersionDurationMs = commandSw.ElapsedMilliseconds; } transaction.Commit(); + + var measures = new Dictionary + { + [TelemetryMeasureName.ReleaseLeasesDurationMs.ToString()] = releaseLeasesDurationMs, + [TelemetryMeasureName.UpdateLastSyncVersionDurationMs.ToString()] = updateLastSyncVersionDurationMs, + }; + + TelemetryInstance.TrackEvent(TelemetryEventName.ReleaseLeasesEnd, this._telemetryProps, measures); } catch (Exception ex) { - this._logger.LogError($"Failed to execute SQL commands to release leases for table '{this._userTable.FullName}' due to exception: {ex.GetType()}." + - $" Exception message: {ex.Message}"); + this._logger.LogError($"Failed to execute SQL commands to release leases for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeases, ex, this._telemetryProps); try { @@ -449,19 +530,19 @@ private async Task ReleaseLeasesAsync(CancellationToken token) catch (Exception ex2) { this._logger.LogError($"Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeasesRollback, ex2, this._telemetryProps); } } } } - } catch (Exception e) { // What should we do if releasing the leases fails? We could try to release them again or just wait, // since eventually the lease time will expire. Then another thread will re-process the same changes // though, so less than ideal. But for now that's the functionality. - this._logger.LogError($"Failed to release leases for table '{this._userTable.FullName}' due to exception: {e.GetType()}." + - $" Exception message: {e.Message}"); + this._logger.LogError($"Failed to release leases for table '{this._userTable.FullName}' due to exception: {e.GetType()}. Exception message: {e.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeases, e, this._telemetryProps); } finally { diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index fd8f82f34..5143fce57 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -3,10 +3,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry; +using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry; using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Listeners; using Microsoft.Data.SqlClient; @@ -32,6 +35,8 @@ internal sealed class SqlTriggerListener : IListener private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; + private readonly IDictionary _telemetryProps; + private SqlTableChangeMonitor _changeMonitor; private int _listenerState; @@ -57,6 +62,11 @@ public SqlTriggerListener(string connectionString, string tableName, string user this._executor = executor; this._logger = logger; this._listenerState = ListenerNotStarted; + + this._telemetryProps = new Dictionary + { + [TelemetryPropertyName.UserFunctionId.ToString()] = this._userFunctionId, + }; } public void Cancel() @@ -71,6 +81,8 @@ public void Dispose() public async Task StartAsync(CancellationToken cancellationToken) { + TelemetryInstance.TrackEvent(TelemetryEventName.StartListenerStart, this._telemetryProps); + int previousState = Interlocked.CompareExchange(ref this._listenerState, ListenerStarting, ListenerNotStarted); switch (previousState) @@ -85,6 +97,7 @@ public async Task StartAsync(CancellationToken cancellationToken) using (var connection = new SqlConnection(this._connectionString)) { await connection.OpenAsync(cancellationToken); + this._telemetryProps.AddConnectionProps(connection); int userTableId = await this.GetUserTableIdAsync(connection, cancellationToken); IReadOnlyList<(string name, string type)> primaryKeyColumns = await this.GetPrimaryKeyColumnsAsync(connection, userTableId, cancellationToken); @@ -92,13 +105,17 @@ public async Task StartAsync(CancellationToken cancellationToken) string workerTableName = string.Format(CultureInfo.InvariantCulture, SqlTriggerConstants.WorkerTableNameFormat, $"{this._userFunctionId}_{userTableId}"); this._logger.LogDebug($"Worker table name: '{workerTableName}'."); + this._telemetryProps[TelemetryPropertyName.WorkerTableName.ToString()] = workerTableName; + + var transactionSw = Stopwatch.StartNew(); + long createdSchemaDurationMs = 0L, createGlobalStateTableDurationMs = 0L, insertGlobalStateTableRowDurationMs = 0L, createWorkerTableDurationMs = 0L; using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) { - await CreateSchemaAsync(connection, transaction, cancellationToken); - await CreateGlobalStateTableAsync(connection, transaction, cancellationToken); - await this.InsertGlobalStateTableRowAsync(connection, transaction, userTableId, cancellationToken); - await CreateWorkerTableAsync(connection, transaction, workerTableName, primaryKeyColumns, cancellationToken); + createdSchemaDurationMs = await CreateSchemaAsync(connection, transaction, cancellationToken); + createGlobalStateTableDurationMs = await CreateGlobalStateTableAsync(connection, transaction, cancellationToken); + insertGlobalStateTableRowDurationMs = await this.InsertGlobalStateTableRowAsync(connection, transaction, userTableId, cancellationToken); + createWorkerTableDurationMs = await CreateWorkerTableAsync(connection, transaction, workerTableName, primaryKeyColumns, cancellationToken); transaction.Commit(); } @@ -114,16 +131,29 @@ public async Task StartAsync(CancellationToken cancellationToken) userTableColumns, primaryKeyColumns.Select(col => col.name).ToList(), this._executor, - this._logger); + this._logger, + this._telemetryProps); this._listenerState = ListenerStarted; this._logger.LogInformation($"Started SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); + + var measures = new Dictionary + { + [TelemetryMeasureName.CreatedSchemaDurationMs.ToString()] = createdSchemaDurationMs, + [TelemetryMeasureName.CreateGlobalStateTableDurationMs.ToString()] = createGlobalStateTableDurationMs, + [TelemetryMeasureName.InsertGlobalStateTableRowDurationMs.ToString()] = insertGlobalStateTableRowDurationMs, + [TelemetryMeasureName.CreateWorkerTableDurationMs.ToString()] = createWorkerTableDurationMs, + [TelemetryMeasureName.TransactionDurationMs.ToString()] = transactionSw.ElapsedMilliseconds, + }; + + TelemetryInstance.TrackEvent(TelemetryEventName.StartListenerEnd, this._telemetryProps, measures); } } catch (Exception ex) { this._listenerState = ListenerNotStarted; this._logger.LogError($"Failed to start SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'. Exception: {ex}"); + TelemetryInstance.TrackException(TelemetryErrorName.StartListener, ex, this._telemetryProps); throw; } @@ -131,6 +161,9 @@ public async Task StartAsync(CancellationToken cancellationToken) public Task StopAsync(CancellationToken cancellationToken) { + TelemetryInstance.TrackEvent(TelemetryEventName.StopListenerStart, this._telemetryProps); + var stopwatch = Stopwatch.StartNew(); + int previousState = Interlocked.CompareExchange(ref this._listenerState, ListenerStopping, ListenerStarted); if (previousState == ListenerStarted) { @@ -140,6 +173,12 @@ public Task StopAsync(CancellationToken cancellationToken) this._logger.LogInformation($"Stopped SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); } + var measures = new Dictionary + { + [TelemetryMeasureName.DurationMs.ToString()] = stopwatch.ElapsedMilliseconds, + }; + + TelemetryInstance.TrackEvent(TelemetryEventName.StopListenerEnd, this._telemetryProps, measures); return Task.CompletedTask; } @@ -259,7 +298,7 @@ private async Task> GetUserTableColumnsAsync(SqlConnection /// /// Creates the schema for global state table and worker tables, if it does not already exist. /// - private static async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) + private static async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { string createSchemaQuery = $@" IF SCHEMA_ID(N'{SqlTriggerConstants.SchemaName}') IS NULL @@ -268,14 +307,16 @@ IF SCHEMA_ID(N'{SqlTriggerConstants.SchemaName}') IS NULL using (var createSchemaCommand = new SqlCommand(createSchemaQuery, connection, transaction)) { + var stopwatch = Stopwatch.StartNew(); await createSchemaCommand.ExecuteNonQueryAsync(cancellationToken); + return stopwatch.ElapsedMilliseconds; } } /// /// Creates the global state table if it does not already exist. /// - private static async Task CreateGlobalStateTableAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) + private static async Task CreateGlobalStateTableAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { string createGlobalStateTableQuery = $@" IF OBJECT_ID(N'{SqlTriggerConstants.GlobalStateTableName}', 'U') IS NULL @@ -289,14 +330,16 @@ PRIMARY KEY (UserFunctionID, UserTableID) using (var createGlobalStateTableCommand = new SqlCommand(createGlobalStateTableQuery, connection, transaction)) { + var stopwatch = Stopwatch.StartNew(); await createGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); + return stopwatch.ElapsedMilliseconds; } } /// /// Inserts row for the 'user function and table' inside the global state table, if one does not already exist. /// - private async Task InsertGlobalStateTableRowAsync(SqlConnection connection, SqlTransaction transaction, int userTableId, CancellationToken cancellationToken) + private async Task InsertGlobalStateTableRowAsync(SqlConnection connection, SqlTransaction transaction, int userTableId, CancellationToken cancellationToken) { object minValidVersion; @@ -329,14 +372,16 @@ INSERT INTO {SqlTriggerConstants.GlobalStateTableName} using (var insertRowGlobalStateTableCommand = new SqlCommand(insertRowGlobalStateTableQuery, connection, transaction)) { + var stopwatch = Stopwatch.StartNew(); await insertRowGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); + return stopwatch.ElapsedMilliseconds; } } /// /// Creates the worker table for the 'user function and table', if one does not already exist. /// - private static async Task CreateWorkerTableAsync( + private static async Task CreateWorkerTableAsync( SqlConnection connection, SqlTransaction transaction, string workerTableName, @@ -359,7 +404,9 @@ PRIMARY KEY ({primaryKeys}) using (var createWorkerTableCommand = new SqlCommand(createWorkerTableQuery, connection, transaction)) { + var stopwatch = Stopwatch.StartNew(); await createWorkerTableCommand.ExecuteNonQueryAsync(cancellationToken); + return stopwatch.ElapsedMilliseconds; } } } From ca410dbb46a9ca8646a3f2775ed4b03a9b0b57ae Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Thu, 11 Aug 2022 00:02:36 +0530 Subject: [PATCH 09/77] Throw exception for unsupported column types (#291) --- Directory.Packages.props | 4 ++-- samples/samples-csharp/packages.lock.json | 23 ++++++++++---------- src/TriggerBinding/SqlTriggerListener.cs | 25 ++++++++++++++++++++-- src/packages.lock.json | 26 +++++++++++------------ test/packages.lock.json | 25 +++++++++++----------- 5 files changed, 63 insertions(+), 40 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 469da8513..ebe10d3e5 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -2,7 +2,7 @@ - + @@ -13,7 +13,7 @@ - + \ No newline at end of file diff --git a/samples/samples-csharp/packages.lock.json b/samples/samples-csharp/packages.lock.json index f184ffc37..c7f928a9c 100644 --- a/samples/samples-csharp/packages.lock.json +++ b/samples/samples-csharp/packages.lock.json @@ -41,9 +41,9 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[13.0.1, )", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + "requested": "[11.0.2, )", + "resolved": "11.0.2", + "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" }, "Azure.Core": { "type": "Transitive", @@ -942,8 +942,8 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==" + "resolved": "4.7.0", + "contentHash": "oJjw3uFuVDJiJNbCD8HB4a2p3NYLdt1fiT5OGsPLw+WTOuG0KpP4OXelMmmVKpClueMsit6xOlzy4wNKQFiBLg==" }, "System.Diagnostics.Tools": { "type": "Transitive", @@ -1727,22 +1727,23 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "2.17.0", + "Microsoft.ApplicationInsights": "2.14.0", "Microsoft.AspNetCore.Http": "2.1.22", "Microsoft.Azure.WebJobs": "3.0.31", "Microsoft.Data.SqlClient": "3.0.1", - "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json": "11.0.2", "System.Runtime.Caching": "4.7.0", "morelinq": "3.3.2" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.17.0, )", - "resolved": "2.17.0", - "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", + "requested": "[2.14.0, )", + "resolved": "2.14.0", + "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "5.0.0" + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory": "4.5.4" } }, "Microsoft.Azure.WebJobs": { diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 5143fce57..4e51e148c 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -278,16 +278,37 @@ FROM sys.indexes AS i /// private async Task> GetUserTableColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { - string getUserTableColumnsQuery = $"SELECT name FROM sys.columns WHERE object_id = {userTableId};"; + string getUserTableColumnsQuery = $@" + SELECT c.name, t.name, t.is_assembly_type + FROM sys.columns AS c + INNER JOIN sys.types AS t ON c.user_type_id = t.user_type_id + WHERE c.object_id = {userTableId}; + "; using (var getUserTableColumnsCommand = new SqlCommand(getUserTableColumnsQuery, connection)) using (SqlDataReader reader = await getUserTableColumnsCommand.ExecuteReaderAsync(cancellationToken)) { var userTableColumns = new List(); + var userDefinedTypeColumns = new List<(string name, string type)>(); while (await reader.ReadAsync(cancellationToken)) { - userTableColumns.Add(reader.GetString(0)); + string columnName = reader.GetString(0); + string columnType = reader.GetString(1); + bool isAssemblyType = reader.GetBoolean(2); + + userTableColumns.Add(columnName); + + if (isAssemblyType) + { + userDefinedTypeColumns.Add((columnName, columnType)); + } + } + + if (userDefinedTypeColumns.Count > 0) + { + string columnNamesAndTypes = string.Join(", ", userDefinedTypeColumns.Select(col => $"'{col.name}' (type: {col.type})")); + throw new InvalidOperationException($"Found column(s) with unsupported type(s): {columnNamesAndTypes} in table: '{this._userTable.FullName}'."); } this._logger.LogDebug($"User table column names: {string.Join(", ", userTableColumns.Select(col => $"'{col}'"))}."); diff --git a/src/packages.lock.json b/src/packages.lock.json index 985d74bb7..48b0acef4 100644 --- a/src/packages.lock.json +++ b/src/packages.lock.json @@ -4,11 +4,12 @@ ".NETStandard,Version=v2.0": { "Microsoft.ApplicationInsights": { "type": "Direct", - "requested": "[2.17.0, )", - "resolved": "2.17.0", - "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", + "requested": "[2.14.0, )", + "resolved": "2.14.0", + "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "5.0.0" + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory": "4.5.4" } }, "Microsoft.AspNetCore.Http": { @@ -93,9 +94,9 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[13.0.1, )", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + "requested": "[11.0.2, )", + "resolved": "11.0.2", + "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" }, "System.Runtime.Caching": { "type": "Direct", @@ -698,11 +699,10 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==", + "resolved": "4.6.0", + "contentHash": "mbBgoR0rRfl2uimsZ2avZY8g7Xnh1Mza0rJZLPcxqiMWlkGukjmRkuMJ/er+AhQuiRIh80CR/Hpeztr80seV5g==", "dependencies": { - "System.Memory": "4.5.4", - "System.Runtime.CompilerServices.Unsafe": "5.0.0" + "System.Memory": "4.5.3" } }, "System.Diagnostics.Process": { @@ -1078,8 +1078,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "ZD9TMpsmYJLrxbbmdvhwt9YEgG5WntEnZ/d1eH8JBX9LBp+Ju8BSBhUGbZMNVHHomWo2KVImJhTDl2hIgw/6MA==" + "resolved": "4.7.0", + "contentHash": "IpU1lcHz8/09yDr9N+Juc7SCgNUz+RohkCQI+KsWKR67XxpFr8Z6c8t1iENCXZuRuNCc4HBwme/MDHNVCwyAKg==" }, "System.Runtime.Extensions": { "type": "Transitive", diff --git a/test/packages.lock.json b/test/packages.lock.json index 114180c95..65443dcee 100644 --- a/test/packages.lock.json +++ b/test/packages.lock.json @@ -51,9 +51,9 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[13.0.1, )", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + "requested": "[11.0.2, )", + "resolved": "11.0.2", + "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" }, "xunit": { "type": "Direct", @@ -1084,8 +1084,8 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==" + "resolved": "4.7.0", + "contentHash": "oJjw3uFuVDJiJNbCD8HB4a2p3NYLdt1fiT5OGsPLw+WTOuG0KpP4OXelMmmVKpClueMsit6xOlzy4wNKQFiBLg==" }, "System.Diagnostics.Tools": { "type": "Transitive", @@ -1930,11 +1930,11 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "2.17.0", + "Microsoft.ApplicationInsights": "2.14.0", "Microsoft.AspNetCore.Http": "2.1.22", "Microsoft.Azure.WebJobs": "3.0.31", "Microsoft.Data.SqlClient": "3.0.1", - "Newtonsoft.Json": "13.0.1", + "Newtonsoft.Json": "11.0.2", "System.Runtime.Caching": "4.7.0", "morelinq": "3.3.2" } @@ -1946,16 +1946,17 @@ "Microsoft.Azure.WebJobs.Extensions.Sql": "99.99.99", "Microsoft.Azure.WebJobs.Extensions.Storage": "5.0.0", "Microsoft.NET.Sdk.Functions": "3.1.1", - "Newtonsoft.Json": "13.0.1" + "Newtonsoft.Json": "11.0.2" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.17.0, )", - "resolved": "2.17.0", - "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", + "requested": "[2.14.0, )", + "resolved": "2.14.0", + "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "5.0.0" + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory": "4.5.4" } }, "Microsoft.Azure.WebJobs": { From d2ffb21f9df872267271ed13af33f423644b37ab Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Thu, 18 Aug 2022 21:38:14 +0530 Subject: [PATCH 10/77] Enable PR validation for triggerbindings branch (#298) --- builds/azure-pipelines/build-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/builds/azure-pipelines/build-pr.yml b/builds/azure-pipelines/build-pr.yml index 5c4829b96..4442edc01 100644 --- a/builds/azure-pipelines/build-pr.yml +++ b/builds/azure-pipelines/build-pr.yml @@ -4,6 +4,7 @@ pr: branches: include: - main + - triggerbindings variables: solution: '**/*.sln' From 5dc95f2afc68f9ee826e8cac8f2413a0be8eb156 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Sat, 20 Aug 2022 10:19:54 +0530 Subject: [PATCH 11/77] Renew all leases through single statement (#297) --- README.md | 2 + src/TriggerBinding/SqlTableChangeMonitor.cs | 75 +++++++------------ .../SqlTriggerBindingProvider.cs | 6 +- src/TriggerBinding/SqlTriggerListener.cs | 4 +- 4 files changed, 35 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 25c56caf1..4c57915bf 100644 --- a/README.md +++ b/README.md @@ -607,6 +607,8 @@ The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sq For more information, please refer to the documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-table). The trigger needs to have read access on the table being monitored for changes as well as to the change tracking system tables. It also needs write access to an `az_func` schema within the database, where it will create additional worker tables to store the trigger states and leases. Each function trigger will thus have an associated change tracking table and worker table. + > **NOTE:** The worker table contains all columns corresponding to the primary key from the user table and three additional columns named `ChangeVersion`, `AttemptCount` and `LeastExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. + #### Trigger Samples The trigger binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/TriggerBinding/SqlTriggerAttribute.cs) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index b23db02b0..3ec7120e0 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -7,7 +7,6 @@ using System.Globalization; using System.IO; using System.Linq; -using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -44,7 +43,6 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly string _workerTableName; private readonly IReadOnlyList _userTableColumns; private readonly IReadOnlyList _primaryKeyColumns; - private readonly IReadOnlyList<(string col, string hash)> _primaryKeyColumnHashes; private readonly IReadOnlyList _rowMatchConditions; private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; @@ -103,11 +101,10 @@ public SqlTableChangeMonitor( this._workerTableName = workerTableName; this._userTableColumns = userTableColumns; this._primaryKeyColumns = primaryKeyColumns; - this._primaryKeyColumnHashes = primaryKeyColumns.Select(col => (col, GetColumnHash(col))).ToList(); // Prep search-conditions that will be used besides WHERE clause to match table rows. this._rowMatchConditions = Enumerable.Range(0, BatchSize) - .Select(index => string.Join(" AND ", this._primaryKeyColumnHashes.Select(ch => $"{ch.col.AsBracketQuotedString()} = @{ch.hash}_{index}"))) + .Select(rowIndex => string.Join(" AND ", this._primaryKeyColumns.Select((col, colIndex) => $"{col.AsBracketQuotedString()} = @{rowIndex}_{colIndex}"))) .ToList(); this._executor = executor; @@ -163,8 +160,8 @@ private async Task RunChangeConsumptionLoopAsync() { if (this._state == State.CheckingForChanges) { - await this.GetChangesAsync(token); - await this.ProcessChangesAsync(token); + await this.GetTableChangesAsync(token); + await this.ProcessTableChangesAsync(token); } await Task.Delay(TimeSpan.FromSeconds(PollingIntervalInSeconds), token); @@ -195,7 +192,7 @@ private async Task RunChangeConsumptionLoopAsync() /// Queries the change/worker tables to check for new changes on the user's table. If any are found, stores the /// change along with the corresponding data from the user table in "_rows". /// - private async Task GetChangesAsync(CancellationToken token) + private async Task GetTableChangesAsync(CancellationToken token) { TelemetryInstance.TrackEvent(TelemetryEventName.GetChangesStart, this._telemetryProps); @@ -292,7 +289,7 @@ private async Task GetChangesAsync(CancellationToken token) } } - private async Task ProcessChangesAsync(CancellationToken token) + private async Task ProcessTableChangesAsync(CancellationToken token) { if (this._rows.Count > 0) { @@ -612,20 +609,6 @@ private static SqlChangeOperation GetChangeOperation(IReadOnlyDictionary - /// Creates GUID string from column name that can be used as name of SQL local variable. The column names - /// cannot be used directly as variable names as they may contain characters like ',', '.', '+', spaces, etc. - /// that are not allowed in variable names. - /// - private static string GetColumnHash(string columnName) - { - using (var sha256 = SHA256.Create()) - { - byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(columnName)); - return new Guid(hash.Take(16).ToArray()).ToString("N"); - } - } - /// /// Builds the command to update the global state table in the case of a new minimum valid version number. /// Sets the LastSyncVersion for this _userTable to be the new minimum valid version number. @@ -699,13 +682,13 @@ private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransa { var acquireLeasesQuery = new StringBuilder(); - for (int index = 0; index < this._rows.Count; index++) + for (int rowIndex = 0; rowIndex < this._rows.Count; rowIndex++) { - string valuesList = string.Join(", ", this._primaryKeyColumnHashes.Select(ch => $"@{ch.hash}_{index}")); - string changeVersion = this._rows[index]["SYS_CHANGE_VERSION"]; + string valuesList = string.Join(", ", this._primaryKeyColumns.Select((_, colIndex) => $"@{rowIndex}_{colIndex}")); + string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"]; acquireLeasesQuery.Append($@" - IF NOT EXISTS (SELECT * FROM {this._workerTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[index]}) + IF NOT EXISTS (SELECT * FROM {this._workerTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}) INSERT INTO {this._workerTableName} WITH (TABLOCKX) VALUES ({valuesList}, {changeVersion}, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME())); ELSE @@ -714,7 +697,7 @@ IF NOT EXISTS (SELECT * FROM {this._workerTableName} WITH (TABLOCKX) WHERE {this ChangeVersion = {changeVersion}, AttemptCount = AttemptCount + 1, LeaseExpirationTime = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) - WHERE {this._rowMatchConditions[index]}; + WHERE {this._rowMatchConditions[rowIndex]}; "); } @@ -728,18 +711,15 @@ IF NOT EXISTS (SELECT * FROM {this._workerTableName} WITH (TABLOCKX) WHERE {this /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildRenewLeasesCommand(SqlConnection connection) { - var renewLeasesQuery = new StringBuilder(); + string matchCondition = string.Join(" OR ", this._rowMatchConditions.Take(this._rows.Count)); - for (int index = 0; index < this._rows.Count; index++) - { - renewLeasesQuery.Append($@" - UPDATE {this._workerTableName} WITH (TABLOCKX) - SET LeaseExpirationTime = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) - WHERE {this._rowMatchConditions[index]}; - "); - } + string renewLeasesQuery = $@" + UPDATE {this._workerTableName} WITH (TABLOCKX) + SET LeaseExpirationTime = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + WHERE {matchCondition}; + "; - return this.GetSqlCommandWithParameters(renewLeasesQuery.ToString(), connection, null); + return this.GetSqlCommandWithParameters(renewLeasesQuery, connection, null); } /// @@ -753,19 +733,19 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa { var releaseLeasesQuery = new StringBuilder("DECLARE @current_change_version bigint;\n"); - for (int index = 0; index < this._rows.Count; index++) + for (int rowIndex = 0; rowIndex < this._rows.Count; rowIndex++) { - string changeVersion = this._rows[index]["SYS_CHANGE_VERSION"]; + string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"]; releaseLeasesQuery.Append($@" SELECT @current_change_version = ChangeVersion FROM {this._workerTableName} WITH (TABLOCKX) - WHERE {this._rowMatchConditions[index]}; + WHERE {this._rowMatchConditions[rowIndex]}; IF @current_change_version <= {changeVersion} - UPDATE {this._workerTableName} WITH (TABLOCKX) + UPDATE {this._workerTableName} WITH (TABLOCKX) SET ChangeVersion = {changeVersion}, AttemptCount = 0, LeaseExpirationTime = NULL - WHERE {this._rowMatchConditions[index]}; + WHERE {this._rowMatchConditions[rowIndex]}; "); } @@ -834,14 +814,11 @@ private SqlCommand GetSqlCommandWithParameters(string commandText, SqlConnection { var command = new SqlCommand(commandText, connection, transaction); - for (int index = 0; index < this._rows.Count; index++) - { - foreach ((string col, string hash) in this._primaryKeyColumnHashes) - { - command.Parameters.Add(new SqlParameter($"@{hash}_{index}", this._rows[index][col])); - } - } + SqlParameter[] parameters = Enumerable.Range(0, this._rows.Count) + .SelectMany(rowIndex => this._primaryKeyColumns.Select((col, colIndex) => new SqlParameter($"@{rowIndex}_{colIndex}", this._rows[rowIndex][col]))) + .ToArray(); + command.Parameters.AddRange(parameters); return command; } diff --git a/src/TriggerBinding/SqlTriggerBindingProvider.cs b/src/TriggerBinding/SqlTriggerBindingProvider.cs index e9754a51c..2fc6644ca 100644 --- a/src/TriggerBinding/SqlTriggerBindingProvider.cs +++ b/src/TriggerBinding/SqlTriggerBindingProvider.cs @@ -57,9 +57,13 @@ public Task TryCreateAsync(TriggerBindingProviderContext contex ParameterInfo parameter = context.Parameter; SqlTriggerAttribute attribute = parameter.GetCustomAttribute(inherit: false); + // During application startup, the WebJobs SDK calls 'TryCreateAsync' method of all registered trigger + // binding providers in sequence for each parameter in the user function. A provider that finds the + // parameter-attribute that it can handle returns the binding object. Rest of the providers are supposed to + // return null. This binding object later gets used for binding before every function invocation. if (attribute == null) { - return Task.FromResult(default(ITriggerBinding)); + return Task.FromResult(null); } if (!IsValidTriggerParameterType(parameter.ParameterType)) diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 6f25cc71a..f99b0f126 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -323,7 +323,7 @@ private static async Task CreateSchemaAsync(SqlConnection connection, SqlT { string createSchemaQuery = $@" IF SCHEMA_ID(N'{SqlTriggerConstants.SchemaName}') IS NULL - EXEC ('CREATE SCHEMA [{SqlTriggerConstants.SchemaName}]'); + EXEC ('CREATE SCHEMA {SqlTriggerConstants.SchemaName}'); "; using (var createSchemaCommand = new SqlCommand(createSchemaQuery, connection, transaction)) @@ -409,7 +409,7 @@ private static async Task CreateWorkerTableAsync( IReadOnlyList<(string name, string type)> primaryKeyColumns, CancellationToken cancellationToken) { - string primaryKeysWithTypes = string.Join(",\n", primaryKeyColumns.Select(col => $"{col.name.AsBracketQuotedString()} [{col.type}]")); + string primaryKeysWithTypes = string.Join(", ", primaryKeyColumns.Select(col => $"{col.name.AsBracketQuotedString()} {col.type}")); string primaryKeys = string.Join(", ", primaryKeyColumns.Select(col => col.name.AsBracketQuotedString())); string createWorkerTableQuery = $@" From f1b273997b692452fa9d49ad2febac2dba22cc81 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Thu, 25 Aug 2022 06:28:27 +0530 Subject: [PATCH 12/77] Add new integration tests for trigger binding (#296) --- .../TriggerBindingSamples/ProductsTrigger.cs | 2 +- src/TriggerBinding/SqlTriggerListener.cs | 23 ++-- test/GlobalSuppressions.cs | 4 + test/Integration/IntegrationTestBase.cs | 32 +++-- .../SqlTriggerBindingIntegrationTests.cs | 110 +++++++++++++++++- ...ductsWithReservedPrimaryKeyColumnNames.sql | 14 +++ .../ProductsWithUnsupportedColumnTypes.sql | 8 ++ .../Tables/ProductsWithoutPrimaryKey.sql | 5 + .../PrimaryKeyNotPresentTrigger.cs | 23 ++++ .../ReservedPrimaryKeyColumnNamesTrigger.cs | 24 ++++ .../test-csharp/TableNotPresentTrigger.cs | 23 ++++ .../UnsupportedColumnTypesTrigger.cs | 23 ++++ ....Azure.WebJobs.Extensions.Sql.Tests.csproj | 8 ++ 13 files changed, 278 insertions(+), 21 deletions(-) create mode 100644 test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql create mode 100644 test/Integration/test-csharp/Database/Tables/ProductsWithUnsupportedColumnTypes.sql create mode 100644 test/Integration/test-csharp/Database/Tables/ProductsWithoutPrimaryKey.sql create mode 100644 test/Integration/test-csharp/PrimaryKeyNotPresentTrigger.cs create mode 100644 test/Integration/test-csharp/ReservedPrimaryKeyColumnNamesTrigger.cs create mode 100644 test/Integration/test-csharp/TableNotPresentTrigger.cs create mode 100644 test/Integration/test-csharp/UnsupportedColumnTypesTrigger.cs diff --git a/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs b/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs index dedaab06d..2a143678f 100644 --- a/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs +++ b/samples/samples-csharp/TriggerBindingSamples/ProductsTrigger.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples { public static class ProductsTrigger { - [FunctionName("ProductsTrigger")] + [FunctionName(nameof(ProductsTrigger))] public static void Run( [SqlTrigger("[dbo].[Products]", ConnectionStringSetting = "SqlConnectionString")] IReadOnlyList> changes, diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index f99b0f126..adab107d8 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -229,21 +229,14 @@ FROM sys.indexes AS i using (var getPrimaryKeyColumnsCommand = new SqlCommand(getPrimaryKeyColumnsQuery, connection)) using (SqlDataReader reader = await getPrimaryKeyColumnsCommand.ExecuteReaderAsync(cancellationToken)) { - string[] reservedColumnNames = new string[] { "ChangeVersion", "AttemptCount", "LeaseExpirationTime" }; - string[] variableLengthTypes = new string[] { "varchar", "nvarchar", "nchar", "char", "binary", "varbinary" }; - string[] variablePrecisionTypes = new string[] { "numeric", "decimal" }; + string[] variableLengthTypes = new[] { "varchar", "nvarchar", "nchar", "char", "binary", "varbinary" }; + string[] variablePrecisionTypes = new[] { "numeric", "decimal" }; var primaryKeyColumns = new List<(string name, string type)>(); while (await reader.ReadAsync(cancellationToken)) { string name = reader.GetString(0); - - if (reservedColumnNames.Contains(name)) - { - throw new InvalidOperationException($"Found reserved column name: '{name}' in table: '{this._userTable.FullName}'."); - } - string type = reader.GetString(1); if (variableLengthTypes.Contains(type, StringComparer.OrdinalIgnoreCase)) @@ -260,7 +253,7 @@ FROM sys.indexes AS i type += $"({precision},{scale})"; } - primaryKeyColumns.Add((name: reader.GetString(0), type)); + primaryKeyColumns.Add((name, type)); } if (primaryKeyColumns.Count == 0) @@ -268,6 +261,16 @@ FROM sys.indexes AS i throw new InvalidOperationException($"Could not find primary key created in table: '{this._userTable.FullName}'."); } + string[] reservedColumnNames = new[] { "ChangeVersion", "AttemptCount", "LeaseExpirationTime" }; + var conflictingColumnNames = primaryKeyColumns.Select(col => col.name).Intersect(reservedColumnNames).ToList(); + + if (conflictingColumnNames.Count > 0) + { + string columnNames = string.Join(", ", conflictingColumnNames.Select(col => $"'{col}'")); + throw new InvalidOperationException($"Found reserved column name(s): {columnNames} in table: '{this._userTable.FullName}'." + + " Please rename them to be able to use trigger binding."); + } + this._logger.LogDebug($"Primary key column names(types): {string.Join(", ", primaryKeyColumns.Select(col => $"'{col.name}({col.type})'"))}."); return primaryKeyColumns; } diff --git a/test/GlobalSuppressions.cs b/test/GlobalSuppressions.cs index 8048b8be3..56034261e 100644 --- a/test/GlobalSuppressions.cs +++ b/test/GlobalSuppressions.cs @@ -13,3 +13,7 @@ [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.AddProductMissingColumns.Run(Microsoft.AspNetCore.Http.HttpRequest,Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common.ProductMissingColumns@)~Microsoft.AspNetCore.Mvc.IActionResult")] [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.AddProductMissingColumnsExceptionFunction.Run(Microsoft.AspNetCore.Http.HttpRequest,Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common.ProductMissingColumns@)~Microsoft.AspNetCore.Mvc.IActionResult")] [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.AddProductsNoPartialUpsert.Run(Microsoft.AspNetCore.Http.HttpRequest,Microsoft.Azure.WebJobs.ICollector{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})~Microsoft.AspNetCore.Mvc.IActionResult")] +[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.PrimaryKeyNotPresentTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] +[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.ReservedPrimaryKeyColumnNamesTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] +[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.TableNotPresentTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] +[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.UnsupportedColumnTypesTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index 9cb181dff..444feedf1 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -164,7 +164,7 @@ protected void StartAzurite() /// - The functionName is different than its route.
/// - You can start multiple functions by passing in a space-separated list of function names.
/// - protected void StartFunctionHost(string functionName, SupportedLanguages language, bool useTestFolder = false) + protected void StartFunctionHost(string functionName, SupportedLanguages language, bool useTestFolder = false, DataReceivedEventHandler customOutputHandler = null) { string workingDirectory = useTestFolder ? GetPathToBin() : Path.Combine(GetPathToBin(), "SqlExtensionSamples", Enum.GetName(typeof(SupportedLanguages), language)); if (!Directory.Exists(workingDirectory)) @@ -188,26 +188,42 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag { StartInfo = startInfo }; + + // Register all handlers before starting the functions host process. + var taskCompletionSource = new TaskCompletionSource(); this.FunctionHost.OutputDataReceived += this.TestOutputHandler; + this.FunctionHost.OutputDataReceived += SignalStartupHandler; + this.FunctionHost.OutputDataReceived += customOutputHandler; + this.FunctionHost.ErrorDataReceived += this.TestOutputHandler; this.FunctionHost.Start(); this.FunctionHost.BeginOutputReadLine(); this.FunctionHost.BeginErrorReadLine(); - var taskCompletionSource = new TaskCompletionSource(); - this.FunctionHost.OutputDataReceived += (object sender, DataReceivedEventArgs e) => + this.TestOutput.WriteLine($"Waiting for Azure Function host to start..."); + + const int FunctionHostStartupTimeoutInSeconds = 60; + bool isCompleted = taskCompletionSource.Task.Wait(TimeSpan.FromSeconds(FunctionHostStartupTimeoutInSeconds)); + Assert.True(isCompleted, "Functions host did not start within specified time."); + + // Give additional time to Functions host to setup routes for the HTTP triggers so that the HTTP requests + // made from the test methods do not get refused. + const int BufferTimeInSeconds = 5; + Task.Delay(TimeSpan.FromSeconds(BufferTimeInSeconds)).Wait(); + + this.TestOutput.WriteLine($"Azure Function host started!"); + this.FunctionHost.OutputDataReceived -= SignalStartupHandler; + + void SignalStartupHandler(object sender, DataReceivedEventArgs e) { // This string is printed after the function host is started up - use this to ensure that we wait long enough // since sometimes the host can take a little while to fully start up - if (e != null && !string.IsNullOrEmpty(e.Data) && e.Data.Contains($"http://localhost:{this.Port}/api")) + if (e.Data?.Contains(" Host initialized ") == true) { taskCompletionSource.SetResult(true); } - }; - this.TestOutput.WriteLine($"Waiting for Azure Function host to start..."); - taskCompletionSource.Task.Wait(60000); - this.TestOutput.WriteLine($"Azure Function host started!"); + } } private static string GetFunctionsCoreToolsPath() diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index e060642a3..42fa6a1f5 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; @@ -21,8 +22,12 @@ public SqlTriggerBindingIntegrationTests(ITestOutputHelper output) : base(output this.EnableChangeTrackingForDatabase(); } + /// + /// Ensures that the user function gets invoked for each of the insert, update and delete operation, and the + /// changes to the user table are passed to the function in correct sequence. + /// [Fact] - public async void BasicTriggerTest() + public async Task SingleOperationTriggerTest() { this.EnableChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); @@ -52,8 +57,12 @@ public async void BasicTriggerTest() } + /// + /// Verifies that if several changes have happened to the table row since last invocation, then a single net + /// change for that row is passed to the user function. + /// [Fact] - public async void MultiOperationTriggerTest() + public async Task MultiOperationTriggerTest() { this.EnableChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); @@ -88,6 +97,69 @@ public async void MultiOperationTriggerTest() changes.Clear(); } + /// + /// Tests the error message when the user table is not present in the database. + /// + [Fact] + public void TableNotPresentTriggerTest() + { + this.StartFunctionsHostAndWaitForError( + nameof(TableNotPresentTrigger), + true, + "Could not find table: 'dbo.TableNotPresent'."); + } + + /// + /// Tests the error message when the user table does not contain primary key. + /// + [Fact] + public void PrimaryKeyNotCreatedTriggerTest() + { + this.StartFunctionsHostAndWaitForError( + nameof(PrimaryKeyNotPresentTrigger), + true, + "Could not find primary key created in table: 'dbo.ProductsWithoutPrimaryKey'."); + } + + /// + /// Tests the error message when the user table contains one or more primary keys with names conflicting with + /// column names in the worker table. + /// + [Fact] + public void ReservedPrimaryKeyColumnNamesTriggerTest() + { + this.StartFunctionsHostAndWaitForError( + nameof(ReservedPrimaryKeyColumnNamesTrigger), + true, + "Found reserved column name(s): 'ChangeVersion', 'AttemptCount', 'LeaseExpirationTime' in table: 'dbo.ProductsWithReservedPrimaryKeyColumnNames'." + + " Please rename them to be able to use trigger binding."); + } + + /// + /// Tests the error message when the user table contains columns of unsupported SQL types. + /// + [Fact] + public void UnsupportedColumnTypesTriggerTest() + { + this.StartFunctionsHostAndWaitForError( + nameof(UnsupportedColumnTypesTrigger), + true, + "Found column(s) with unsupported type(s): 'Location' (type: geography), 'Geometry' (type: geometry), 'Organization' (type: hierarchyid)" + + " in table: 'dbo.ProductsWithUnsupportedColumnTypes'."); + } + + /// + /// Tests the error message when change tracking is not enabled on the user table. + /// + [Fact] + public void ChangeTrackingNotEnabledTriggerTest() + { + this.StartFunctionsHostAndWaitForError( + nameof(ProductsTrigger), + false, + "Could not find change tracking enabled for table: 'dbo.Products'."); + } + private void EnableChangeTrackingForDatabase() { this.ExecuteNonQuery($@" @@ -163,5 +235,39 @@ private static void ValidateProductChanges(List> changes, int id += 1; } } + + /// + /// Launches the functions runtime host, waits for it to encounter error while starting the SQL trigger listener, + /// and asserts that the logged error message matches with the supplied error message. + /// + /// Name of the user function that should cause error in trigger listener + /// Whether the functions host should be launched from test folder + /// Expected error message string + private void StartFunctionsHostAndWaitForError(string functionName, bool useTestFolder, string expectedErrorMessage) + { + string errorMessage = null; + var tcs = new TaskCompletionSource(); + + void OutputHandler(object sender, DataReceivedEventArgs e) + { + if (errorMessage == null && e.Data?.Contains("Failed to start SQL trigger listener") == true) + { + // SQL trigger listener throws exception of type InvalidOperationException for all error conditions. + string exceptionPrefix = "Exception: System.InvalidOperationException: "; + int index = e.Data.IndexOf(exceptionPrefix, StringComparison.Ordinal); + Assert.NotEqual(-1, index); + + errorMessage = e.Data[(index + exceptionPrefix.Length)..]; + tcs.SetResult(true); + } + }; + + // All trigger integration tests are only using C# functions for testing at the moment. + this.StartFunctionHost(functionName, Common.SupportedLanguages.CSharp, useTestFolder, OutputHandler); + this.FunctionHost.OutputDataReceived -= OutputHandler; + this.FunctionHost.Kill(); + + Assert.Equal(expectedErrorMessage, errorMessage); + } } } \ No newline at end of file diff --git a/test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql b/test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql new file mode 100644 index 000000000..032b4a745 --- /dev/null +++ b/test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql @@ -0,0 +1,14 @@ +CREATE TABLE [ProductsWithReservedPrimaryKeyColumnNames] ( + [ProductId] [int] NOT NULL IDENTITY(1, 1), + [ChangeVersion] [int] NOT NULL, + [AttemptCount] [int] NOT NULL, + [LeaseExpirationTime] [int] NOT NULL, + [Name] [nvarchar](100) NULL, + [Cost] [int] NULL, + PRIMARY KEY ( + ProductId, + ChangeVersion, + AttemptCount, + LeaseExpirationTime + ) +); \ No newline at end of file diff --git a/test/Integration/test-csharp/Database/Tables/ProductsWithUnsupportedColumnTypes.sql b/test/Integration/test-csharp/Database/Tables/ProductsWithUnsupportedColumnTypes.sql new file mode 100644 index 000000000..74eb540d8 --- /dev/null +++ b/test/Integration/test-csharp/Database/Tables/ProductsWithUnsupportedColumnTypes.sql @@ -0,0 +1,8 @@ +CREATE TABLE [ProductsWithUnsupportedColumnTypes] ( + [ProductId] [int] NOT NULL PRIMARY KEY, + [Name] [nvarchar](100) NULL, + [Cost] [int] NULL, + [Location] [geography] NULL, + [Geometry] [geometry] NULL, + [Organization] [hierarchyid] NULL +); \ No newline at end of file diff --git a/test/Integration/test-csharp/Database/Tables/ProductsWithoutPrimaryKey.sql b/test/Integration/test-csharp/Database/Tables/ProductsWithoutPrimaryKey.sql new file mode 100644 index 000000000..c2735b410 --- /dev/null +++ b/test/Integration/test-csharp/Database/Tables/ProductsWithoutPrimaryKey.sql @@ -0,0 +1,5 @@ +CREATE TABLE [ProductsWithoutPrimaryKey] ( + [ProductId] [int] NOT NULL, + [Name] [nvarchar](100) NULL, + [Cost] [int] NULL +); \ No newline at end of file diff --git a/test/Integration/test-csharp/PrimaryKeyNotPresentTrigger.cs b/test/Integration/test-csharp/PrimaryKeyNotPresentTrigger.cs new file mode 100644 index 000000000..2bdf41085 --- /dev/null +++ b/test/Integration/test-csharp/PrimaryKeyNotPresentTrigger.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration +{ + public static class PrimaryKeyNotPresentTrigger + { + /// + /// Used in verification of the error message when the user table does not contain primary key. + /// + [FunctionName(nameof(PrimaryKeyNotPresentTrigger))] + public static void Run( + [SqlTrigger("[dbo].[ProductsWithoutPrimaryKey]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> products) + { + throw new NotImplementedException("Associated test case should fail before the function is invoked."); + } + } +} diff --git a/test/Integration/test-csharp/ReservedPrimaryKeyColumnNamesTrigger.cs b/test/Integration/test-csharp/ReservedPrimaryKeyColumnNamesTrigger.cs new file mode 100644 index 000000000..1389c279f --- /dev/null +++ b/test/Integration/test-csharp/ReservedPrimaryKeyColumnNamesTrigger.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration +{ + public static class ReservedPrimaryKeyColumnNamesTrigger + { + /// + /// Used in verification of the error message when the user table contains one or more primary keys with names + /// conflicting with column names in the worker table. + /// + [FunctionName(nameof(ReservedPrimaryKeyColumnNamesTrigger))] + public static void Run( + [SqlTrigger("[dbo].[ProductsWithReservedPrimaryKeyColumnNames]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> products) + { + throw new NotImplementedException("Associated test case should fail before the function is invoked."); + } + } +} diff --git a/test/Integration/test-csharp/TableNotPresentTrigger.cs b/test/Integration/test-csharp/TableNotPresentTrigger.cs new file mode 100644 index 000000000..345becf53 --- /dev/null +++ b/test/Integration/test-csharp/TableNotPresentTrigger.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration +{ + public static class TableNotPresentTrigger + { + /// + /// Used in verification of the error message when the user table is not present in the database. + /// + [FunctionName(nameof(TableNotPresentTrigger))] + public static void Run( + [SqlTrigger("[dbo].[TableNotPresent]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> products) + { + throw new NotImplementedException("Associated test case should fail before the function is invoked."); + } + } +} diff --git a/test/Integration/test-csharp/UnsupportedColumnTypesTrigger.cs b/test/Integration/test-csharp/UnsupportedColumnTypesTrigger.cs new file mode 100644 index 000000000..7b15fc817 --- /dev/null +++ b/test/Integration/test-csharp/UnsupportedColumnTypesTrigger.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration +{ + public static class UnsupportedColumnTypesTrigger + { + /// + /// Used in verification of the error message when the user table contains columns of unsupported SQL types. + /// + [FunctionName(nameof(UnsupportedColumnTypesTrigger))] + public static void Run( + [SqlTrigger("[dbo].[ProductsWithUnsupportedColumnTypes]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> products) + { + throw new NotImplementedException("Associated test case should fail before the function is invoked."); + } + } +} diff --git a/test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj b/test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj index 77f6efd95..54c504783 100644 --- a/test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj +++ b/test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj @@ -33,4 +33,12 @@ + + + <_CSharpTestSqlFiles Include="Integration\test-csharp\Database\**\*.*" /> + + + + + From 2bf43883279e580bf08bced0647bad6299960aa8 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Thu, 25 Aug 2022 20:26:31 +0530 Subject: [PATCH 13/77] Do not open extra SQL connections (#318) --- src/TriggerBinding/SqlTableChangeMonitor.cs | 210 ++++++++++---------- 1 file changed, 100 insertions(+), 110 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 3ec7120e0..21fa0bb39 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -160,8 +160,8 @@ private async Task RunChangeConsumptionLoopAsync() { if (this._state == State.CheckingForChanges) { - await this.GetTableChangesAsync(token); - await this.ProcessTableChangesAsync(token); + await this.GetTableChangesAsync(connection, token); + await this.ProcessTableChangesAsync(connection, token); } await Task.Delay(TimeSpan.FromSeconds(PollingIntervalInSeconds), token); @@ -192,89 +192,84 @@ private async Task RunChangeConsumptionLoopAsync() /// Queries the change/worker tables to check for new changes on the user's table. If any are found, stores the /// change along with the corresponding data from the user table in "_rows". /// - private async Task GetTableChangesAsync(CancellationToken token) + private async Task GetTableChangesAsync(SqlConnection connection, CancellationToken token) { TelemetryInstance.TrackEvent(TelemetryEventName.GetChangesStart, this._telemetryProps); try { - using (var connection = new SqlConnection(this._connectionString)) - { - await connection.OpenAsync(token); + var transactionSw = Stopwatch.StartNew(); + long setLastSyncVersionDurationMs = 0L, getChangesDurationMs = 0L, acquireLeasesDurationMs = 0L; - var transactionSw = Stopwatch.StartNew(); - long setLastSyncVersionDurationMs = 0L, getChangesDurationMs = 0L, acquireLeasesDurationMs = 0L; - - using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + { + try { - try + // Update the version number stored in the global state table if necessary before using it. + using (SqlCommand updateTablesPreInvocationCommand = this.BuildUpdateTablesPreInvocation(connection, transaction)) { - // Update the version number stored in the global state table if necessary before using it. - using (SqlCommand updateTablesPreInvocationCommand = this.BuildUpdateTablesPreInvocation(connection, transaction)) - { - var commandSw = Stopwatch.StartNew(); - await updateTablesPreInvocationCommand.ExecuteNonQueryAsync(token); - setLastSyncVersionDurationMs = commandSw.ElapsedMilliseconds; - } + var commandSw = Stopwatch.StartNew(); + await updateTablesPreInvocationCommand.ExecuteNonQueryAsync(token); + setLastSyncVersionDurationMs = commandSw.ElapsedMilliseconds; + } - // Use the version number to query for new changes. - using (SqlCommand getChangesCommand = this.BuildGetChangesCommand(connection, transaction)) - { - var commandSw = Stopwatch.StartNew(); - var rows = new List>(); + // Use the version number to query for new changes. + using (SqlCommand getChangesCommand = this.BuildGetChangesCommand(connection, transaction)) + { + var commandSw = Stopwatch.StartNew(); + var rows = new List>(); - using (SqlDataReader reader = await getChangesCommand.ExecuteReaderAsync(token)) + using (SqlDataReader reader = await getChangesCommand.ExecuteReaderAsync(token)) + { + while (await reader.ReadAsync(token)) { - while (await reader.ReadAsync(token)) - { - rows.Add(SqlBindingUtilities.BuildDictionaryFromSqlRow(reader)); - } + rows.Add(SqlBindingUtilities.BuildDictionaryFromSqlRow(reader)); } - - this._rows = rows; - getChangesDurationMs = commandSw.ElapsedMilliseconds; } - this._logger.LogDebug($"Changed rows count: {this._rows.Count}."); + this._rows = rows; + getChangesDurationMs = commandSw.ElapsedMilliseconds; + } + + this._logger.LogDebug($"Changed rows count: {this._rows.Count}."); - // If changes were found, acquire leases on them. - if (this._rows.Count > 0) + // If changes were found, acquire leases on them. + if (this._rows.Count > 0) + { + using (SqlCommand acquireLeasesCommand = this.BuildAcquireLeasesCommand(connection, transaction)) { - using (SqlCommand acquireLeasesCommand = this.BuildAcquireLeasesCommand(connection, transaction)) - { - var commandSw = Stopwatch.StartNew(); - await acquireLeasesCommand.ExecuteNonQueryAsync(token); - acquireLeasesDurationMs = commandSw.ElapsedMilliseconds; - } + var commandSw = Stopwatch.StartNew(); + await acquireLeasesCommand.ExecuteNonQueryAsync(token); + acquireLeasesDurationMs = commandSw.ElapsedMilliseconds; } + } - transaction.Commit(); + transaction.Commit(); - var measures = new Dictionary - { - [TelemetryMeasureName.SetLastSyncVersionDurationMs] = setLastSyncVersionDurationMs, - [TelemetryMeasureName.GetChangesDurationMs] = getChangesDurationMs, - [TelemetryMeasureName.AcquireLeasesDurationMs] = acquireLeasesDurationMs, - [TelemetryMeasureName.TransactionDurationMs] = transactionSw.ElapsedMilliseconds, - [TelemetryMeasureName.BatchCount] = this._rows.Count, - }; - - TelemetryInstance.TrackEvent(TelemetryEventName.GetChangesEnd, this._telemetryProps, measures); - } - catch (Exception ex) + var measures = new Dictionary { - this._logger.LogError($"Failed to query list of changes for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); - TelemetryInstance.TrackException(TelemetryErrorName.GetChanges, ex, this._telemetryProps); + [TelemetryMeasureName.SetLastSyncVersionDurationMs] = setLastSyncVersionDurationMs, + [TelemetryMeasureName.GetChangesDurationMs] = getChangesDurationMs, + [TelemetryMeasureName.AcquireLeasesDurationMs] = acquireLeasesDurationMs, + [TelemetryMeasureName.TransactionDurationMs] = transactionSw.ElapsedMilliseconds, + [TelemetryMeasureName.BatchCount] = this._rows.Count, + }; - try - { - transaction.Rollback(); - } - catch (Exception ex2) - { - this._logger.LogError($"Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}"); - TelemetryInstance.TrackException(TelemetryErrorName.GetChangesRollback, ex2, this._telemetryProps); - } + TelemetryInstance.TrackEvent(TelemetryEventName.GetChangesEnd, this._telemetryProps, measures); + } + catch (Exception ex) + { + this._logger.LogError($"Failed to query list of changes for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.GetChanges, ex, this._telemetryProps); + + try + { + transaction.Rollback(); + } + catch (Exception ex2) + { + this._logger.LogError($"Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.GetChangesRollback, ex2, this._telemetryProps); } } } @@ -289,7 +284,7 @@ private async Task GetTableChangesAsync(CancellationToken token) } } - private async Task ProcessTableChangesAsync(CancellationToken token) + private async Task ProcessTableChangesAsync(SqlConnection connection, CancellationToken token) { if (this._rows.Count > 0) { @@ -329,7 +324,7 @@ private async Task ProcessTableChangesAsync(CancellationToken token) if (result.Succeeded) { TelemetryInstance.TrackEvent(TelemetryEventName.TriggerFunctionEnd, this._telemetryProps, measures); - await this.ReleaseLeasesAsync(token); + await this.ReleaseLeasesAsync(connection, token); } else { @@ -467,7 +462,7 @@ private async Task ClearRowsAsync(bool acquireLock) /// Releases the leases held on "_rows". /// /// - private async Task ReleaseLeasesAsync(CancellationToken token) + private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToken token) { TelemetryInstance.TrackEvent(TelemetryEventName.ReleaseLeasesStart, this._telemetryProps); @@ -477,58 +472,53 @@ private async Task ReleaseLeasesAsync(CancellationToken token) try { - using (var connection = new SqlConnection(this._connectionString)) - { - await connection.OpenAsync(token); + var transactionSw = Stopwatch.StartNew(); + long releaseLeasesDurationMs = 0L, updateLastSyncVersionDurationMs = 0L; - var transactionSw = Stopwatch.StartNew(); - long releaseLeasesDurationMs = 0L, updateLastSyncVersionDurationMs = 0L; - - using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + { + try { - try + // Release the leases held on "_rows". + using (SqlCommand releaseLeasesCommand = this.BuildReleaseLeasesCommand(connection, transaction)) { - // Release the leases held on "_rows". - using (SqlCommand releaseLeasesCommand = this.BuildReleaseLeasesCommand(connection, transaction)) - { - var commandSw = Stopwatch.StartNew(); - await releaseLeasesCommand.ExecuteNonQueryAsync(token); - releaseLeasesDurationMs = commandSw.ElapsedMilliseconds; - } + var commandSw = Stopwatch.StartNew(); + await releaseLeasesCommand.ExecuteNonQueryAsync(token); + releaseLeasesDurationMs = commandSw.ElapsedMilliseconds; + } - // Update the global state table if we have processed all changes with ChangeVersion <= newLastSyncVersion, - // and clean up the worker table to remove all rows with ChangeVersion <= newLastSyncVersion. - using (SqlCommand updateTablesPostInvocationCommand = this.BuildUpdateTablesPostInvocation(connection, transaction, newLastSyncVersion)) - { - var commandSw = Stopwatch.StartNew(); - await updateTablesPostInvocationCommand.ExecuteNonQueryAsync(token); - updateLastSyncVersionDurationMs = commandSw.ElapsedMilliseconds; - } + // Update the global state table if we have processed all changes with ChangeVersion <= newLastSyncVersion, + // and clean up the worker table to remove all rows with ChangeVersion <= newLastSyncVersion. + using (SqlCommand updateTablesPostInvocationCommand = this.BuildUpdateTablesPostInvocation(connection, transaction, newLastSyncVersion)) + { + var commandSw = Stopwatch.StartNew(); + await updateTablesPostInvocationCommand.ExecuteNonQueryAsync(token); + updateLastSyncVersionDurationMs = commandSw.ElapsedMilliseconds; + } - transaction.Commit(); + transaction.Commit(); - var measures = new Dictionary - { - [TelemetryMeasureName.ReleaseLeasesDurationMs] = releaseLeasesDurationMs, - [TelemetryMeasureName.UpdateLastSyncVersionDurationMs] = updateLastSyncVersionDurationMs, - }; + var measures = new Dictionary + { + [TelemetryMeasureName.ReleaseLeasesDurationMs] = releaseLeasesDurationMs, + [TelemetryMeasureName.UpdateLastSyncVersionDurationMs] = updateLastSyncVersionDurationMs, + }; + + TelemetryInstance.TrackEvent(TelemetryEventName.ReleaseLeasesEnd, this._telemetryProps, measures); + } + catch (Exception ex) + { + this._logger.LogError($"Failed to execute SQL commands to release leases for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeases, ex, this._telemetryProps); - TelemetryInstance.TrackEvent(TelemetryEventName.ReleaseLeasesEnd, this._telemetryProps, measures); + try + { + transaction.Rollback(); } - catch (Exception ex) + catch (Exception ex2) { - this._logger.LogError($"Failed to execute SQL commands to release leases for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); - TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeases, ex, this._telemetryProps); - - try - { - transaction.Rollback(); - } - catch (Exception ex2) - { - this._logger.LogError($"Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}"); - TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeasesRollback, ex2, this._telemetryProps); - } + this._logger.LogError($"Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeasesRollback, ex2, this._telemetryProps); } } } From 7833025925149e848ab34522a0ea4028fd5577ab Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:29:17 +0530 Subject: [PATCH 14/77] Use constants for worker table column names (#310) --- README.md | 2 +- src/TriggerBinding/SqlTableChangeMonitor.cs | 32 +++++++++++-------- src/TriggerBinding/SqlTriggerConstants.cs | 4 +++ src/TriggerBinding/SqlTriggerListener.cs | 14 +++++--- .../SqlTriggerBindingIntegrationTests.cs | 9 ++++-- ...ductsWithReservedPrimaryKeyColumnNames.sql | 12 +++---- 6 files changed, 47 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 4c57915bf..bcf44b59b 100644 --- a/README.md +++ b/README.md @@ -607,7 +607,7 @@ The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sq For more information, please refer to the documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-table). The trigger needs to have read access on the table being monitored for changes as well as to the change tracking system tables. It also needs write access to an `az_func` schema within the database, where it will create additional worker tables to store the trigger states and leases. Each function trigger will thus have an associated change tracking table and worker table. - > **NOTE:** The worker table contains all columns corresponding to the primary key from the user table and three additional columns named `ChangeVersion`, `AttemptCount` and `LeastExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. + > **NOTE:** The worker table contains all columns corresponding to the primary key from the user table and three additional columns named `_az_func_ChangeVersion`, `_az_func_AttemptCount` and `_az_func_LeastExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. #### Trigger Samples The trigger binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/TriggerBinding/SqlTriggerAttribute.cs) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 21fa0bb39..a9d36d7c8 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -647,14 +647,15 @@ private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransacti SELECT TOP {BatchSize} {selectList}, c.SYS_CHANGE_VERSION, c.SYS_CHANGE_OPERATION, - w.ChangeVersion, w.AttemptCount, w.LeaseExpirationTime + w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName}, w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName}, w.{SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCondition} WHERE - (w.LeaseExpirationTime IS NULL AND (w.ChangeVersion IS NULL OR w.ChangeVersion < c.SYS_CHANGE_VERSION) OR - w.LeaseExpirationTime < SYSDATETIME()) AND - (w.AttemptCount IS NULL OR w.AttemptCount < {MaxAttemptCount}) + (w.{SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} IS NULL AND + (w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName} IS NULL OR w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName} < c.SYS_CHANGE_VERSION) OR + w.{SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND + (w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName} IS NULL OR w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName} < {MaxAttemptCount}) ORDER BY c.SYS_CHANGE_VERSION ASC; "; @@ -684,9 +685,9 @@ IF NOT EXISTS (SELECT * FROM {this._workerTableName} WITH (TABLOCKX) WHERE {this ELSE UPDATE {this._workerTableName} WITH (TABLOCKX) SET - ChangeVersion = {changeVersion}, - AttemptCount = AttemptCount + 1, - LeaseExpirationTime = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + {SqlTriggerConstants.WorkerTableChangeVersionColumnName} = {changeVersion}, + {SqlTriggerConstants.WorkerTableAttemptCountColumnName} = {SqlTriggerConstants.WorkerTableAttemptCountColumnName} + 1, + {SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) WHERE {this._rowMatchConditions[rowIndex]}; "); } @@ -705,7 +706,7 @@ private SqlCommand BuildRenewLeasesCommand(SqlConnection connection) string renewLeasesQuery = $@" UPDATE {this._workerTableName} WITH (TABLOCKX) - SET LeaseExpirationTime = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + SET {SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) WHERE {matchCondition}; "; @@ -728,13 +729,16 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"]; releaseLeasesQuery.Append($@" - SELECT @current_change_version = ChangeVersion + SELECT @current_change_version = {SqlTriggerConstants.WorkerTableChangeVersionColumnName} FROM {this._workerTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}; IF @current_change_version <= {changeVersion} UPDATE {this._workerTableName} WITH (TABLOCKX) - SET ChangeVersion = {changeVersion}, AttemptCount = 0, LeaseExpirationTime = NULL + SET + {SqlTriggerConstants.WorkerTableChangeVersionColumnName} = {changeVersion}, + {SqlTriggerConstants.WorkerTableAttemptCountColumnName} = 0, + {SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} = NULL WHERE {this._rowMatchConditions[rowIndex]}; "); } @@ -768,8 +772,10 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_ LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} WHERE c.SYS_CHANGE_VERSION <= {newLastSyncVersion} AND - ((w.ChangeVersion IS NULL OR w.ChangeVersion != c.SYS_CHANGE_VERSION OR w.LeaseExpirationTime IS NOT NULL) AND - (w.AttemptCount IS NULL OR w.AttemptCount < {MaxAttemptCount}))) AS Changes + ((w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName} IS NULL OR + w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName} != c.SYS_CHANGE_VERSION OR + w.{SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} IS NOT NULL) AND + (w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName} IS NULL OR w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName} < {MaxAttemptCount}))) AS Changes IF @unprocessed_changes = 0 AND @current_last_sync_version < {newLastSyncVersion} BEGIN @@ -777,7 +783,7 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_ SET LastSyncVersion = {newLastSyncVersion} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; - DELETE FROM {this._workerTableName} WITH (TABLOCKX) WHERE ChangeVersion <= {newLastSyncVersion}; + DELETE FROM {this._workerTableName} WITH (TABLOCKX) WHERE {SqlTriggerConstants.WorkerTableChangeVersionColumnName} <= {newLastSyncVersion}; END "; diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index 6aa13358d..3b8b828e1 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -10,5 +10,9 @@ internal static class SqlTriggerConstants public const string GlobalStateTableName = "[" + SchemaName + "].[GlobalState]"; public const string WorkerTableNameFormat = "[" + SchemaName + "].[Worker_{0}]"; + + public const string WorkerTableChangeVersionColumnName = "_az_func_ChangeVersion"; + public const string WorkerTableAttemptCountColumnName = "_az_func_AttemptCount"; + public const string WorkerTableLeaseExpirationTimeColumnName = "_az_func_LeaseExpirationTime"; } } \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index adab107d8..7c5cd4169 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -261,7 +261,13 @@ FROM sys.indexes AS i throw new InvalidOperationException($"Could not find primary key created in table: '{this._userTable.FullName}'."); } - string[] reservedColumnNames = new[] { "ChangeVersion", "AttemptCount", "LeaseExpirationTime" }; + string[] reservedColumnNames = new[] + { + SqlTriggerConstants.WorkerTableChangeVersionColumnName, + SqlTriggerConstants.WorkerTableAttemptCountColumnName, + SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName + }; + var conflictingColumnNames = primaryKeyColumns.Select(col => col.name).Intersect(reservedColumnNames).ToList(); if (conflictingColumnNames.Count > 0) @@ -419,9 +425,9 @@ private static async Task CreateWorkerTableAsync( IF OBJECT_ID(N'{workerTableName}', 'U') IS NULL CREATE TABLE {workerTableName} ( {primaryKeysWithTypes}, - ChangeVersion bigint NOT NULL, - AttemptCount int NOT NULL, - LeaseExpirationTime datetime2, + {SqlTriggerConstants.WorkerTableChangeVersionColumnName} bigint NOT NULL, + {SqlTriggerConstants.WorkerTableAttemptCountColumnName} int NOT NULL, + {SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} datetime2, PRIMARY KEY ({primaryKeys}) ); "; diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 42fa6a1f5..758ea303f 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -131,7 +131,7 @@ public void ReservedPrimaryKeyColumnNamesTriggerTest() this.StartFunctionsHostAndWaitForError( nameof(ReservedPrimaryKeyColumnNamesTrigger), true, - "Found reserved column name(s): 'ChangeVersion', 'AttemptCount', 'LeaseExpirationTime' in table: 'dbo.ProductsWithReservedPrimaryKeyColumnNames'." + + "Found reserved column name(s): '_az_func_ChangeVersion', '_az_func_AttemptCount', '_az_func_LeaseExpirationTime' in table: 'dbo.ProductsWithReservedPrimaryKeyColumnNames'." + " Please rename them to be able to use trigger binding."); } @@ -223,8 +223,13 @@ private static void ValidateProductChanges(List> changes, int int count = last_id - first_id + 1; Assert.Equal(count, changes.Count); + // Since the table rows are changed with a single SQL statement, the changes are not guaranteed to arrive in + // ProductID-order. Occasionally, we find the items in the second batch are passed to the user function in + // reverse order, which is an expected behavior. + IEnumerable> orderedChanges = changes.OrderBy(change => change.Item.ProductID); + int id = first_id; - foreach (SqlChange change in changes) + foreach (SqlChange change in orderedChanges) { Assert.Equal(operation, change.Operation); Product product = change.Item; diff --git a/test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql b/test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql index 032b4a745..ac05fb59a 100644 --- a/test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql +++ b/test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql @@ -1,14 +1,14 @@ CREATE TABLE [ProductsWithReservedPrimaryKeyColumnNames] ( [ProductId] [int] NOT NULL IDENTITY(1, 1), - [ChangeVersion] [int] NOT NULL, - [AttemptCount] [int] NOT NULL, - [LeaseExpirationTime] [int] NOT NULL, + [_az_func_ChangeVersion] [int] NOT NULL, + [_az_func_AttemptCount] [int] NOT NULL, + [_az_func_LeaseExpirationTime] [int] NOT NULL, [Name] [nvarchar](100) NULL, [Cost] [int] NULL, PRIMARY KEY ( ProductId, - ChangeVersion, - AttemptCount, - LeaseExpirationTime + _az_func_ChangeVersion, + _az_func_AttemptCount, + _az_func_LeaseExpirationTime ) ); \ No newline at end of file From d46795654e355ce5ba77191382d56bad83998854 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Fri, 26 Aug 2022 07:06:39 +0530 Subject: [PATCH 15/77] Rename worker table to leases table (#319) --- README.md | 4 +- src/Telemetry/Telemetry.cs | 4 +- src/TriggerBinding/SqlTableChangeMonitor.cs | 78 +++++++++---------- src/TriggerBinding/SqlTriggerConstants.cs | 8 +- src/TriggerBinding/SqlTriggerListener.cs | 46 +++++------ .../SqlTriggerBindingIntegrationTests.cs | 2 +- .../ReservedPrimaryKeyColumnNamesTrigger.cs | 2 +- 7 files changed, 72 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index bcf44b59b..b7a4c5654 100644 --- a/README.md +++ b/README.md @@ -605,9 +605,9 @@ The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sq ENABLE CHANGE_TRACKING; ``` - For more information, please refer to the documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-table). The trigger needs to have read access on the table being monitored for changes as well as to the change tracking system tables. It also needs write access to an `az_func` schema within the database, where it will create additional worker tables to store the trigger states and leases. Each function trigger will thus have an associated change tracking table and worker table. + For more information, please refer to the documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-table). The trigger needs to have read access on the table being monitored for changes as well as to the change tracking system tables. It also needs write access to an `az_func` schema within the database, where it will create additional leases tables to store the trigger states and leases. Each function trigger will thus have an associated change tracking table and leases table. - > **NOTE:** The worker table contains all columns corresponding to the primary key from the user table and three additional columns named `_az_func_ChangeVersion`, `_az_func_AttemptCount` and `_az_func_LeastExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. + > **NOTE:** The leases table contains all columns corresponding to the primary key from the user table and three additional columns named `_az_func_ChangeVersion`, `_az_func_AttemptCount` and `_az_func_LeastExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. #### Trigger Samples The trigger binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/TriggerBinding/SqlTriggerAttribute.cs) diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index 56723c835..5874551c9 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -359,11 +359,11 @@ public enum TelemetryPropertyName ErrorName, ExceptionType, HasIdentityColumn, + LeasesTableName, QueryType, ServerVersion, Type, UserFunctionId, - WorkerTableName, } /// @@ -376,7 +376,7 @@ public enum TelemetryMeasureName CommandDurationMs, CreatedSchemaDurationMs, CreateGlobalStateTableDurationMs, - CreateWorkerTableDurationMs, + CreateLeasesTableDurationMs, DurationMs, GetCaseSensitivityDurationMs, GetChangesDurationMs, diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index a9d36d7c8..2f64b85be 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -40,7 +40,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly int _userTableId; private readonly SqlObject _userTable; private readonly string _userFunctionId; - private readonly string _workerTableName; + private readonly string _leasesTableName; private readonly IReadOnlyList _userTableColumns; private readonly IReadOnlyList _primaryKeyColumns; private readonly IReadOnlyList _rowMatchConditions; @@ -67,7 +67,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable /// SQL object ID of the user table /// instance created with user table name /// Unique identifier for the user function - /// Name of the worker table + /// Name of the leases table /// List of all column names in the user table /// List of primary key column names in the user table /// Defines contract for triggering user function @@ -78,7 +78,7 @@ public SqlTableChangeMonitor( int userTableId, SqlObject userTable, string userFunctionId, - string workerTableName, + string leasesTableName, IReadOnlyList userTableColumns, IReadOnlyList primaryKeyColumns, ITriggeredFunctionExecutor executor, @@ -88,7 +88,7 @@ public SqlTableChangeMonitor( _ = !string.IsNullOrEmpty(connectionString) ? true : throw new ArgumentNullException(nameof(connectionString)); _ = !string.IsNullOrEmpty(userTable.FullName) ? true : throw new ArgumentNullException(nameof(userTable)); _ = !string.IsNullOrEmpty(userFunctionId) ? true : throw new ArgumentNullException(nameof(userFunctionId)); - _ = !string.IsNullOrEmpty(workerTableName) ? true : throw new ArgumentNullException(nameof(workerTableName)); + _ = !string.IsNullOrEmpty(leasesTableName) ? true : throw new ArgumentNullException(nameof(leasesTableName)); _ = userTableColumns ?? throw new ArgumentNullException(nameof(userTableColumns)); _ = primaryKeyColumns ?? throw new ArgumentNullException(nameof(primaryKeyColumns)); _ = executor ?? throw new ArgumentNullException(nameof(executor)); @@ -98,7 +98,7 @@ public SqlTableChangeMonitor( this._userTableId = userTableId; this._userTable = userTable; this._userFunctionId = userFunctionId; - this._workerTableName = workerTableName; + this._leasesTableName = leasesTableName; this._userTableColumns = userTableColumns; this._primaryKeyColumns = primaryKeyColumns; @@ -137,7 +137,7 @@ public void Dispose() /// /// Executed once every period. If the state of the change monitor is - /// , then the method query the change/worker tables for changes on the + /// , then the method query the change/leases tables for changes on the /// user's table. If any are found, the state of the change monitor is transitioned to /// and the user's function is executed with the found changes. If the /// execution is successful, the leases on "_rows" are released and the state transitions to @@ -189,7 +189,7 @@ private async Task RunChangeConsumptionLoopAsync() } /// - /// Queries the change/worker tables to check for new changes on the user's table. If any are found, stores the + /// Queries the change/leases tables to check for new changes on the user's table. If any are found, stores the /// change along with the corresponding data from the user table in "_rows". /// private async Task GetTableChangesAsync(SqlConnection connection, CancellationToken token) @@ -386,7 +386,7 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken if (this._state == State.ProcessingChanges) { // I don't think I need a transaction for renewing leases. If this worker reads in a row from the - // worker table and determines that it corresponds to its batch of changes, but then that row gets + // leases table and determines that it corresponds to its batch of changes, but then that row gets // deleted by a cleanup task, it shouldn't renew the lease on it anyways. using (SqlCommand renewLeasesCommand = this.BuildRenewLeasesCommand(connection)) { @@ -488,7 +488,7 @@ private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToke } // Update the global state table if we have processed all changes with ChangeVersion <= newLastSyncVersion, - // and clean up the worker table to remove all rows with ChangeVersion <= newLastSyncVersion. + // and clean up the leases table to remove all rows with ChangeVersion <= newLastSyncVersion. using (SqlCommand updateTablesPostInvocationCommand = this.BuildUpdateTablesPostInvocation(connection, transaction, newLastSyncVersion)) { var commandSw = Stopwatch.StartNew(); @@ -584,7 +584,7 @@ private IReadOnlyList> GetChanges() /// /// Gets the change associated with this row (either an insert, update or delete). /// - /// The (combined) row from the change table and worker table + /// The (combined) row from the change table and leases table /// Thrown if the value of the "SYS_CHANGE_OPERATION" column is none of "I", "U", or "D" /// SqlChangeOperation.Insert for an insert, SqlChangeOperation.Update for an update, and SqlChangeOperation.Delete for a delete private static SqlChangeOperation GetChangeOperation(IReadOnlyDictionary row) @@ -636,7 +636,7 @@ private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransacti { string selectList = string.Join(", ", this._userTableColumns.Select(col => this._primaryKeyColumns.Contains(col) ? $"c.{col.AsBracketQuotedString()}" : $"u.{col.AsBracketQuotedString()}")); string userTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = u.{col.AsBracketQuotedString()}")); - string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = w.{col.AsBracketQuotedString()}")); + string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = l.{col.AsBracketQuotedString()}")); string getChangesQuery = $@" DECLARE @last_sync_version bigint; @@ -647,15 +647,15 @@ private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransacti SELECT TOP {BatchSize} {selectList}, c.SYS_CHANGE_VERSION, c.SYS_CHANGE_OPERATION, - w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName}, w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName}, w.{SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} + l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName}, l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName}, l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c - LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} + LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCondition} WHERE - (w.{SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} IS NULL AND - (w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName} IS NULL OR w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName} < c.SYS_CHANGE_VERSION) OR - w.{SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND - (w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName} IS NULL OR w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName} < {MaxAttemptCount}) + (l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NULL AND + (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.SYS_CHANGE_VERSION) OR + l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND + (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxAttemptCount}) ORDER BY c.SYS_CHANGE_VERSION ASC; "; @@ -679,15 +679,15 @@ private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransa string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"]; acquireLeasesQuery.Append($@" - IF NOT EXISTS (SELECT * FROM {this._workerTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}) - INSERT INTO {this._workerTableName} WITH (TABLOCKX) + IF NOT EXISTS (SELECT * FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}) + INSERT INTO {this._leasesTableName} WITH (TABLOCKX) VALUES ({valuesList}, {changeVersion}, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME())); ELSE - UPDATE {this._workerTableName} WITH (TABLOCKX) + UPDATE {this._leasesTableName} WITH (TABLOCKX) SET - {SqlTriggerConstants.WorkerTableChangeVersionColumnName} = {changeVersion}, - {SqlTriggerConstants.WorkerTableAttemptCountColumnName} = {SqlTriggerConstants.WorkerTableAttemptCountColumnName} + 1, - {SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + {SqlTriggerConstants.LeasesTableChangeVersionColumnName} = {changeVersion}, + {SqlTriggerConstants.LeasesTableAttemptCountColumnName} = {SqlTriggerConstants.LeasesTableAttemptCountColumnName} + 1, + {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) WHERE {this._rowMatchConditions[rowIndex]}; "); } @@ -705,8 +705,8 @@ private SqlCommand BuildRenewLeasesCommand(SqlConnection connection) string matchCondition = string.Join(" OR ", this._rowMatchConditions.Take(this._rows.Count)); string renewLeasesQuery = $@" - UPDATE {this._workerTableName} WITH (TABLOCKX) - SET {SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + UPDATE {this._leasesTableName} WITH (TABLOCKX) + SET {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) WHERE {matchCondition}; "; @@ -729,16 +729,16 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"]; releaseLeasesQuery.Append($@" - SELECT @current_change_version = {SqlTriggerConstants.WorkerTableChangeVersionColumnName} - FROM {this._workerTableName} WITH (TABLOCKX) + SELECT @current_change_version = {SqlTriggerConstants.LeasesTableChangeVersionColumnName} + FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}; IF @current_change_version <= {changeVersion} - UPDATE {this._workerTableName} WITH (TABLOCKX) + UPDATE {this._leasesTableName} WITH (TABLOCKX) SET - {SqlTriggerConstants.WorkerTableChangeVersionColumnName} = {changeVersion}, - {SqlTriggerConstants.WorkerTableAttemptCountColumnName} = 0, - {SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} = NULL + {SqlTriggerConstants.LeasesTableChangeVersionColumnName} = {changeVersion}, + {SqlTriggerConstants.LeasesTableAttemptCountColumnName} = 0, + {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = NULL WHERE {this._rowMatchConditions[rowIndex]}; "); } @@ -748,7 +748,7 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa /// /// Builds the command to update the global version number in _globalStateTable after successful invocation of - /// the user's function. If the global version number is updated, also cleans the worker table and removes all + /// the user's function. If the global version number is updated, also cleans the leases table and removes all /// rows for which ChangeVersion <= newLastSyncVersion. /// /// The connection to add to the returned SqlCommand @@ -757,7 +757,7 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, SqlTransaction transaction, long newLastSyncVersion) { - string workerTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = w.{col.AsBracketQuotedString()}")); + string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = l.{col.AsBracketQuotedString()}")); string updateTablesPostInvocationQuery = $@" DECLARE @current_last_sync_version bigint; @@ -769,13 +769,13 @@ private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, Sql SELECT @unprocessed_changes = COUNT(*) FROM ( SELECT c.SYS_CHANGE_VERSION FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_sync_version) AS c - LEFT OUTER JOIN {this._workerTableName} AS w WITH (TABLOCKX) ON {workerTableJoinCondition} + LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} WHERE c.SYS_CHANGE_VERSION <= {newLastSyncVersion} AND - ((w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName} IS NULL OR - w.{SqlTriggerConstants.WorkerTableChangeVersionColumnName} != c.SYS_CHANGE_VERSION OR - w.{SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} IS NOT NULL) AND - (w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName} IS NULL OR w.{SqlTriggerConstants.WorkerTableAttemptCountColumnName} < {MaxAttemptCount}))) AS Changes + ((l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR + l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} != c.SYS_CHANGE_VERSION OR + l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NOT NULL) AND + (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxAttemptCount}))) AS Changes IF @unprocessed_changes = 0 AND @current_last_sync_version < {newLastSyncVersion} BEGIN @@ -783,7 +783,7 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_ SET LastSyncVersion = {newLastSyncVersion} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; - DELETE FROM {this._workerTableName} WITH (TABLOCKX) WHERE {SqlTriggerConstants.WorkerTableChangeVersionColumnName} <= {newLastSyncVersion}; + DELETE FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {SqlTriggerConstants.LeasesTableChangeVersionColumnName} <= {newLastSyncVersion}; END "; diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index 3b8b828e1..2f37d866f 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -9,10 +9,10 @@ internal static class SqlTriggerConstants public const string GlobalStateTableName = "[" + SchemaName + "].[GlobalState]"; - public const string WorkerTableNameFormat = "[" + SchemaName + "].[Worker_{0}]"; + public const string LeasesTableNameFormat = "[" + SchemaName + "].[Leases_{0}]"; - public const string WorkerTableChangeVersionColumnName = "_az_func_ChangeVersion"; - public const string WorkerTableAttemptCountColumnName = "_az_func_AttemptCount"; - public const string WorkerTableLeaseExpirationTimeColumnName = "_az_func_LeaseExpirationTime"; + public const string LeasesTableChangeVersionColumnName = "_az_func_ChangeVersion"; + public const string LeasesTableAttemptCountColumnName = "_az_func_AttemptCount"; + public const string LeasesTableLeaseExpirationTimeColumnName = "_az_func_LeaseExpirationTime"; } } \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 7c5cd4169..789ed05e9 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -103,19 +103,19 @@ public async Task StartAsync(CancellationToken cancellationToken) IReadOnlyList<(string name, string type)> primaryKeyColumns = await this.GetPrimaryKeyColumnsAsync(connection, userTableId, cancellationToken); IReadOnlyList userTableColumns = await this.GetUserTableColumnsAsync(connection, userTableId, cancellationToken); - string workerTableName = string.Format(CultureInfo.InvariantCulture, SqlTriggerConstants.WorkerTableNameFormat, $"{this._userFunctionId}_{userTableId}"); - this._logger.LogDebug($"Worker table name: '{workerTableName}'."); - this._telemetryProps[TelemetryPropertyName.WorkerTableName] = workerTableName; + string leasesTableName = string.Format(CultureInfo.InvariantCulture, SqlTriggerConstants.LeasesTableNameFormat, $"{this._userFunctionId}_{userTableId}"); + this._logger.LogDebug($"leases table name: '{leasesTableName}'."); + this._telemetryProps[TelemetryPropertyName.LeasesTableName] = leasesTableName; var transactionSw = Stopwatch.StartNew(); - long createdSchemaDurationMs = 0L, createGlobalStateTableDurationMs = 0L, insertGlobalStateTableRowDurationMs = 0L, createWorkerTableDurationMs = 0L; + long createdSchemaDurationMs = 0L, createGlobalStateTableDurationMs = 0L, insertGlobalStateTableRowDurationMs = 0L, createLeasesTableDurationMs = 0L; using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) { createdSchemaDurationMs = await CreateSchemaAsync(connection, transaction, cancellationToken); createGlobalStateTableDurationMs = await CreateGlobalStateTableAsync(connection, transaction, cancellationToken); insertGlobalStateTableRowDurationMs = await this.InsertGlobalStateTableRowAsync(connection, transaction, userTableId, cancellationToken); - createWorkerTableDurationMs = await CreateWorkerTableAsync(connection, transaction, workerTableName, primaryKeyColumns, cancellationToken); + createLeasesTableDurationMs = await CreateLeasesTableAsync(connection, transaction, leasesTableName, primaryKeyColumns, cancellationToken); transaction.Commit(); } @@ -127,7 +127,7 @@ public async Task StartAsync(CancellationToken cancellationToken) userTableId, this._userTable, this._userFunctionId, - workerTableName, + leasesTableName, userTableColumns, primaryKeyColumns.Select(col => col.name).ToList(), this._executor, @@ -142,7 +142,7 @@ public async Task StartAsync(CancellationToken cancellationToken) [TelemetryMeasureName.CreatedSchemaDurationMs] = createdSchemaDurationMs, [TelemetryMeasureName.CreateGlobalStateTableDurationMs] = createGlobalStateTableDurationMs, [TelemetryMeasureName.InsertGlobalStateTableRowDurationMs] = insertGlobalStateTableRowDurationMs, - [TelemetryMeasureName.CreateWorkerTableDurationMs] = createWorkerTableDurationMs, + [TelemetryMeasureName.CreateLeasesTableDurationMs] = createLeasesTableDurationMs, [TelemetryMeasureName.TransactionDurationMs] = transactionSw.ElapsedMilliseconds, }; @@ -213,7 +213,7 @@ private async Task GetUserTableIdAsync(SqlConnection connection, Cancellati /// Gets the names and types of primary key columns of the user table. /// /// - /// Thrown if there are no primary key columns present in the user table or if their names conflict with columns in worker table. + /// Thrown if there are no primary key columns present in the user table or if their names conflict with columns in leases table. /// private async Task> GetPrimaryKeyColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { @@ -263,9 +263,9 @@ FROM sys.indexes AS i string[] reservedColumnNames = new[] { - SqlTriggerConstants.WorkerTableChangeVersionColumnName, - SqlTriggerConstants.WorkerTableAttemptCountColumnName, - SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName + SqlTriggerConstants.LeasesTableChangeVersionColumnName, + SqlTriggerConstants.LeasesTableAttemptCountColumnName, + SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName }; var conflictingColumnNames = primaryKeyColumns.Select(col => col.name).Intersect(reservedColumnNames).ToList(); @@ -326,7 +326,7 @@ FROM sys.columns AS c } /// - /// Creates the schema for global state table and worker tables, if it does not already exist. + /// Creates the schema for global state table and leases tables, if it does not already exist. /// private static async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { @@ -409,33 +409,33 @@ INSERT INTO {SqlTriggerConstants.GlobalStateTableName} } /// - /// Creates the worker table for the 'user function and table', if one does not already exist. + /// Creates the leases table for the 'user function and table', if one does not already exist. /// - private static async Task CreateWorkerTableAsync( + private static async Task CreateLeasesTableAsync( SqlConnection connection, SqlTransaction transaction, - string workerTableName, + string leasesTableName, IReadOnlyList<(string name, string type)> primaryKeyColumns, CancellationToken cancellationToken) { string primaryKeysWithTypes = string.Join(", ", primaryKeyColumns.Select(col => $"{col.name.AsBracketQuotedString()} {col.type}")); string primaryKeys = string.Join(", ", primaryKeyColumns.Select(col => col.name.AsBracketQuotedString())); - string createWorkerTableQuery = $@" - IF OBJECT_ID(N'{workerTableName}', 'U') IS NULL - CREATE TABLE {workerTableName} ( + string createLeasesTableQuery = $@" + IF OBJECT_ID(N'{leasesTableName}', 'U') IS NULL + CREATE TABLE {leasesTableName} ( {primaryKeysWithTypes}, - {SqlTriggerConstants.WorkerTableChangeVersionColumnName} bigint NOT NULL, - {SqlTriggerConstants.WorkerTableAttemptCountColumnName} int NOT NULL, - {SqlTriggerConstants.WorkerTableLeaseExpirationTimeColumnName} datetime2, + {SqlTriggerConstants.LeasesTableChangeVersionColumnName} bigint NOT NULL, + {SqlTriggerConstants.LeasesTableAttemptCountColumnName} int NOT NULL, + {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} datetime2, PRIMARY KEY ({primaryKeys}) ); "; - using (var createWorkerTableCommand = new SqlCommand(createWorkerTableQuery, connection, transaction)) + using (var createLeasesTableCommand = new SqlCommand(createLeasesTableQuery, connection, transaction)) { var stopwatch = Stopwatch.StartNew(); - await createWorkerTableCommand.ExecuteNonQueryAsync(cancellationToken); + await createLeasesTableCommand.ExecuteNonQueryAsync(cancellationToken); return stopwatch.ElapsedMilliseconds; } } diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 758ea303f..c92ab7a83 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -123,7 +123,7 @@ public void PrimaryKeyNotCreatedTriggerTest() /// /// Tests the error message when the user table contains one or more primary keys with names conflicting with - /// column names in the worker table. + /// column names in the leases table. /// [Fact] public void ReservedPrimaryKeyColumnNamesTriggerTest() diff --git a/test/Integration/test-csharp/ReservedPrimaryKeyColumnNamesTrigger.cs b/test/Integration/test-csharp/ReservedPrimaryKeyColumnNamesTrigger.cs index 1389c279f..98f67662c 100644 --- a/test/Integration/test-csharp/ReservedPrimaryKeyColumnNamesTrigger.cs +++ b/test/Integration/test-csharp/ReservedPrimaryKeyColumnNamesTrigger.cs @@ -11,7 +11,7 @@ public static class ReservedPrimaryKeyColumnNamesTrigger { /// /// Used in verification of the error message when the user table contains one or more primary keys with names - /// conflicting with column names in the worker table. + /// conflicting with column names in the leases table. /// [FunctionName(nameof(ReservedPrimaryKeyColumnNamesTrigger))] public static void Run( From d5f2036d70008820bb47b5faaa26928300ba12c8 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Mon, 29 Aug 2022 10:06:58 -0700 Subject: [PATCH 16/77] Fix spelling (#322) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b7a4c5654..1610e404b 100644 --- a/README.md +++ b/README.md @@ -607,7 +607,7 @@ The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sq For more information, please refer to the documentation [here](https://docs.microsoft.com/sql/relational-databases/track-changes/enable-and-disable-change-tracking-sql-server#enable-change-tracking-for-a-table). The trigger needs to have read access on the table being monitored for changes as well as to the change tracking system tables. It also needs write access to an `az_func` schema within the database, where it will create additional leases tables to store the trigger states and leases. Each function trigger will thus have an associated change tracking table and leases table. - > **NOTE:** The leases table contains all columns corresponding to the primary key from the user table and three additional columns named `_az_func_ChangeVersion`, `_az_func_AttemptCount` and `_az_func_LeastExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. + > **NOTE:** The leases table contains all columns corresponding to the primary key from the user table and three additional columns named `_az_func_ChangeVersion`, `_az_func_AttemptCount` and `_az_func_LeaseExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. #### Trigger Samples The trigger binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/TriggerBinding/SqlTriggerAttribute.cs) From 3330551c14fdd8c11956b3120ff76afba337d552 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Mon, 29 Aug 2022 20:15:25 -0700 Subject: [PATCH 17/77] Fix duplicate key error when exceptions occur (#323) --- src/TriggerBinding/SqlTriggerListener.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 789ed05e9..46ac1e708 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -35,7 +35,7 @@ internal sealed class SqlTriggerListener : IListener private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; - private readonly IDictionary _telemetryProps; + private readonly IDictionary _telemetryProps = new Dictionary(); private SqlTableChangeMonitor _changeMonitor; private int _listenerState; @@ -62,11 +62,6 @@ public SqlTriggerListener(string connectionString, string tableName, string user this._executor = executor; this._logger = logger; this._listenerState = ListenerNotStarted; - - this._telemetryProps = new Dictionary - { - [TelemetryPropertyName.UserFunctionId] = this._userFunctionId, - }; } public void Cancel() @@ -81,6 +76,7 @@ public void Dispose() public async Task StartAsync(CancellationToken cancellationToken) { + this.InitializeTelemetryProps(); TelemetryInstance.TrackEvent(TelemetryEventName.StartListenerStart, this._telemetryProps); int previousState = Interlocked.CompareExchange(ref this._listenerState, ListenerStarting, ListenerNotStarted); @@ -439,5 +435,14 @@ PRIMARY KEY ({primaryKeys}) return stopwatch.ElapsedMilliseconds; } } + + /// + /// Clears the current telemetry property dictionary and initializes the default initial properties. + /// + private void InitializeTelemetryProps() + { + this._telemetryProps.Clear(); + this._telemetryProps[TelemetryPropertyName.UserFunctionId] = this._userFunctionId; + } } } \ No newline at end of file From d544da9f8685073100b1b4c8cdea3c0e06c828cc Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 30 Aug 2022 08:29:00 -0700 Subject: [PATCH 18/77] Fix trigger tutorial location --- README.md | 87 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 0391ca9b6..2a411138d 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,13 @@ Azure SQL bindings for Azure Functions are supported for: - [.NET functions](#net-functions) - [Input Binding Tutorial](#input-binding-tutorial) - [Output Binding Tutorial](#output-binding-tutorial) + - [Trigger Binding Tutorial](#trigger-binding-tutorial) + - [JavaScript functions](#javascript-functions) + - [Input Binding Tutorial](#input-binding-tutorial-1) + - [Output Binding Tutorial](#output-binding-tutorial-1) + - [Python functions](#python-functions) + - [Input Binding Tutorial](#input-binding-tutorial-2) + - [Output Binding Tutorial](#output-binding-tutorial-2) - [More Samples](#more-samples) - [Input Binding](#input-binding) - [Query String](#query-string) @@ -297,6 +304,46 @@ Note: This tutorial requires that a SQL database is setup as shown in [Create a - Hit 'F5' to run your code. Click the link to upsert the output array values in your SQL table. Your upserted values should launch in the browser. - Congratulations! You have successfully created your first SQL output binding! Checkout [Output Binding](#Output-Binding) for more information on how to use it and explore on your own! +#### Trigger Binding Tutorial + +Note: This tutorial requires that a SQL database is setup as shown in [Create a SQL Server](#create-a-sql-server), and that you have the 'Employee.cs' file from the [Input Binding Tutorial](#input-binding-tutorial). + +- Create a new file with the following content: + + ```csharp + using System.Collections.Generic; + using Microsoft.Azure.WebJobs; + using Microsoft.Extensions.Logging; + using Microsoft.Azure.WebJobs.Extensions.Sql; + + namespace Company.Function + { + public static class EmployeeTrigger + { + [FunctionName("EmployeeTrigger")] + public static void Run( + [SqlTrigger("[dbo].[Employees]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> changes, + ILogger logger) + { + foreach (SqlChange change in changes) + { + Employee employee = change.Item; + logger.LogInformation($"Change operation: {change.Operation}"); + logger.LogInformation($"EmployeeID: {employee.EmployeeId}, FirstName: {employee.FirstName}, LastName: {employee.LastName}, Company: {employee.Company}, Team: {employee.Team}"); + } + } + } + } + ``` + +- *Skip these steps if you have not completed the output binding tutorial.* + - Open your output binding file and modify some of the values. For example, change the value of Team column from 'Functions' to 'Azure SQL'. + - Hit 'F5' to run your code. Click the link of the HTTP trigger from the output binding tutorial. +- Update, insert, or delete rows in your SQL table while the function app is running and observe the function logs. +- You should see the new log messages in the Visual Studio Code terminal containing the values of row-columns after the update operation. +- Congratulations! You have successfully created your first SQL trigger binding! Checkout [Trigger Samples](#trigger-samples) for more information on how to use it and explore on your own! + ### JavaScript functions @@ -484,46 +531,6 @@ Note: This tutorial requires that a SQL database is setup as shown in [Create a - Hit 'F5' to run your code. Click the link to upsert the output array values in your SQL table. Your upserted values should launch in the browser. - Congratulations! You have successfully created your first SQL output binding! Checkout [Output Binding](#Output-Binding) for more information on how to use it and explore on your own! -### Trigger Binding Tutorial - -Note: This tutorial requires that a SQL database is setup as shown in [Create a SQL Server](#create-a-sql-server), and that you have the 'Employee.cs' file from the [Input Binding Tutorial](#input-binding-tutorial). - -- Create a new file with the following content: - - ```csharp - using System.Collections.Generic; - using Microsoft.Azure.WebJobs; - using Microsoft.Extensions.Logging; - using Microsoft.Azure.WebJobs.Extensions.Sql; - - namespace Company.Function - { - public static class EmployeeTrigger - { - [FunctionName("EmployeeTrigger")] - public static void Run( - [SqlTrigger("[dbo].[Employees]", ConnectionStringSetting = "SqlConnectionString")] - IReadOnlyList> changes, - ILogger logger) - { - foreach (SqlChange change in changes) - { - Employee employee = change.Item; - logger.LogInformation($"Change operation: {change.Operation}"); - logger.LogInformation($"EmployeeID: {employee.EmployeeId}, FirstName: {employee.FirstName}, LastName: {employee.LastName}, Company: {employee.Company}, Team: {employee.Team}"); - } - } - } - } - ``` - -- *Skip these steps if you have not completed the output binding tutorial.* - - Open your output binding file and modify some of the values. For example, change the value of Team column from 'Functions' to 'Azure SQL'. - - Hit 'F5' to run your code. Click the link of the HTTP trigger from the output binding tutorial. -- Update, insert, or delete rows in your SQL table while the function app is running and observe the function logs. -- You should see the new log messages in the Visual Studio Code terminal containing the values of row-columns after the update operation. -- Congratulations! You have successfully created your first SQL trigger binding! Checkout [Trigger Samples](#trigger-samples) for more information on how to use it and explore on your own! - ## More Samples ### Input Binding From 0d3a0579ced1920fac9376de60950b6182c301de Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 30 Aug 2022 08:35:22 -0700 Subject: [PATCH 19/77] Fix README --- README.md | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2a411138d..6b1246fbf 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Azure SQL bindings for Azure Functions are supported for: - [Docker container](#docker-container) - [Azure SQL Database](#azure-sql-database) - [SQL Setup](#sql-setup) - - [Create .NET Function App](#create-net-function-app) + - [Create a Function App](#create-a-function-app) - [Configure Function App](#configure-function-app) - [Tutorials](#tutorials) - [.NET functions](#net-functions) @@ -103,9 +103,9 @@ ALTER TABLE ['{table_name}'] ADD CONSTRAINT PKey PRIMARY KEY CLUSTERED (['{prima ``` -### Create .NET Function App +### Create a Function App -Now you will need a .NET Function App to add the binding to. If you have one created already you can skip this step. +Now you will need a Function App to add the binding to. If you have one created already you can skip this step. These steps can be done in the Terminal/CLI or with PowerShell. @@ -120,9 +120,32 @@ These steps can be done in the Terminal/CLI or with PowerShell. func init --worker-runtime dotnet ``` -**Note**: Other languages are not supported at this time + **JavaScript (NodeJS)** + ```bash + mkdir MyApp + cd MyApp + func init --worker-runtime node --language javascript + ``` + + **TypeScript (NodeJS)** + ```bash + mkdir MyApp + cd MyApp + func init --worker-runtime node --language typescript + ``` + + **Python** + + *See [#250](https://github.com/Azure/azure-functions-sql-extension/issues/250) before starting.* + ```bash + mkdir MyApp + cd MyApp + func init --worker-runtime python + ``` + +3. Enable SQL bindings on the function app. More information can be found [in Microsoft Docs](https://docs.microsoft.com/azure/azure-functions/functions-bindings-azure-sql). -1. Install the extension. + **.NET:** Install the extension. ```powershell dotnet add package Microsoft.Azure.WebJobs.Extensions.Sql --prerelease From 74901da1e89965b1a37304b9da1d5d3dc6c0f83a Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 30 Aug 2022 09:43:58 -0700 Subject: [PATCH 20/77] Add more debug logging (#324) * Add more debug logging * Move startlistener event * Move comment --- src/TriggerBinding/SqlTableChangeMonitor.cs | 62 ++++++++++++----- src/TriggerBinding/SqlTriggerListener.cs | 73 ++++++++++++++++----- src/Utils.cs | 6 ++ 3 files changed, 106 insertions(+), 35 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 2f64b85be..d0f8241f0 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -145,7 +145,7 @@ public void Dispose() /// private async Task RunChangeConsumptionLoopAsync() { - this._logger.LogInformation("Starting change consumption loop."); + this._logger.LogDebugWithThreadId("Starting change consumption loop."); try { @@ -153,17 +153,19 @@ private async Task RunChangeConsumptionLoopAsync() using (var connection = new SqlConnection(this._connectionString)) { + this._logger.LogDebugWithThreadId("BEGIN OpenChangeConsumptionConnection"); await connection.OpenAsync(token); - + this._logger.LogDebugWithThreadId("END OpenChangeConsumptionConnection"); // Check for cancellation request only after a cycle of checking and processing of changes completes. while (!token.IsCancellationRequested) { + this._logger.LogDebugWithThreadId($"BEGIN CheckingForChanges State={this._state}"); if (this._state == State.CheckingForChanges) { await this.GetTableChangesAsync(connection, token); await this.ProcessTableChangesAsync(connection, token); } - + this._logger.LogDebugWithThreadId("END CheckingForChanges"); await Task.Delay(TimeSpan.FromSeconds(PollingIntervalInSeconds), token); } } @@ -195,7 +197,7 @@ private async Task RunChangeConsumptionLoopAsync() private async Task GetTableChangesAsync(SqlConnection connection, CancellationToken token) { TelemetryInstance.TrackEvent(TelemetryEventName.GetChangesStart, this._telemetryProps); - + this._logger.LogDebugWithThreadId("BEGIN GetTableChanges"); try { var transactionSw = Stopwatch.StartNew(); @@ -208,14 +210,17 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo // Update the version number stored in the global state table if necessary before using it. using (SqlCommand updateTablesPreInvocationCommand = this.BuildUpdateTablesPreInvocation(connection, transaction)) { + this._logger.LogDebugWithThreadId($"BEGIN UpdateTablesPreInvocation Query={updateTablesPreInvocationCommand.CommandText}"); var commandSw = Stopwatch.StartNew(); await updateTablesPreInvocationCommand.ExecuteNonQueryAsync(token); setLastSyncVersionDurationMs = commandSw.ElapsedMilliseconds; } + this._logger.LogDebugWithThreadId($"END UpdateTablesPreInvocation Duration={setLastSyncVersionDurationMs}ms"); // Use the version number to query for new changes. using (SqlCommand getChangesCommand = this.BuildGetChangesCommand(connection, transaction)) { + this._logger.LogDebugWithThreadId($"BEGIN GetChanges Query={getChangesCommand.CommandText}"); var commandSw = Stopwatch.StartNew(); var rows = new List>(); @@ -230,18 +235,19 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo this._rows = rows; getChangesDurationMs = commandSw.ElapsedMilliseconds; } - - this._logger.LogDebug($"Changed rows count: {this._rows.Count}."); + this._logger.LogDebugWithThreadId($"END GetChanges Duration={getChangesDurationMs}ms ChangedRows={this._rows.Count}"); // If changes were found, acquire leases on them. if (this._rows.Count > 0) { using (SqlCommand acquireLeasesCommand = this.BuildAcquireLeasesCommand(connection, transaction)) { + this._logger.LogDebugWithThreadId($"BEGIN AcquireLeases Query={acquireLeasesCommand.CommandText}"); var commandSw = Stopwatch.StartNew(); await acquireLeasesCommand.ExecuteNonQueryAsync(token); acquireLeasesDurationMs = commandSw.ElapsedMilliseconds; } + this._logger.LogDebugWithThreadId($"END AcquireLeases Duration={acquireLeasesDurationMs}ms"); } transaction.Commit(); @@ -282,10 +288,12 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo this._logger.LogError($"Failed to check for changes in table '{this._userTable.FullName}' due to exception: {e.GetType()}. Exception message: {e.Message}"); TelemetryInstance.TrackException(TelemetryErrorName.GetChanges, e, this._telemetryProps); } + this._logger.LogDebugWithThreadId("END GetTableChanges"); } private async Task ProcessTableChangesAsync(SqlConnection connection, CancellationToken token) { + this._logger.LogDebugWithThreadId("BEGIN ProcessTableChanges"); if (this._rows.Count > 0) { this._state = State.ProcessingChanges; @@ -297,7 +305,7 @@ private async Task ProcessTableChangesAsync(SqlConnection connection, Cancellati // thing. We could still try to trigger on the correctly processed changes, but that adds additional // complication because we don't want to release the leases on the incorrectly processed changes. // For now, just give up I guess? - changes = this.GetChanges(); + changes = this.ProcessChanges(); } catch (Exception e) { @@ -311,18 +319,20 @@ private async Task ProcessTableChangesAsync(SqlConnection connection, Cancellati var input = new TriggeredFunctionData() { TriggerValue = changes }; TelemetryInstance.TrackEvent(TelemetryEventName.TriggerFunctionStart, this._telemetryProps); + this._logger.LogDebugWithThreadId("Executing triggered function"); var stopwatch = Stopwatch.StartNew(); FunctionResult result = await this._executor.TryExecuteAsync(input, this._cancellationTokenSourceExecutor.Token); - + long durationMs = stopwatch.ElapsedMilliseconds; var measures = new Dictionary { - [TelemetryMeasureName.DurationMs] = stopwatch.ElapsedMilliseconds, + [TelemetryMeasureName.DurationMs] = durationMs, [TelemetryMeasureName.BatchCount] = this._rows.Count, }; if (result.Succeeded) { + this._logger.LogDebugWithThreadId($"Successfully triggered function. Duration={durationMs}ms"); TelemetryInstance.TrackEvent(TelemetryEventName.TriggerFunctionEnd, this._telemetryProps, measures); await this.ReleaseLeasesAsync(connection, token); } @@ -337,6 +347,7 @@ private async Task ProcessTableChangesAsync(SqlConnection connection, Cancellati } } } + this._logger.LogDebugWithThreadId("END ProcessTableChanges"); } /// @@ -353,11 +364,14 @@ private async void RunLeaseRenewalLoopAsync() using (var connection = new SqlConnection(this._connectionString)) { + this._logger.LogDebugWithThreadId("BEGIN OpenLeaseRenewalLoopConnection"); await connection.OpenAsync(token); - + this._logger.LogDebugWithThreadId("END OpenLeaseRenewalLoopConnection"); while (!token.IsCancellationRequested) { + this._logger.LogDebugWithThreadId("BEGIN WaitRowsLock - LeaseRenewal"); await this._rowsLock.WaitAsync(token); + this._logger.LogDebugWithThreadId("END WaitRowsLock - LeaseRenewal"); await this.RenewLeasesAsync(connection, token); await Task.Delay(TimeSpan.FromSeconds(LeaseRenewalIntervalInSeconds), token); } @@ -391,13 +405,16 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken using (SqlCommand renewLeasesCommand = this.BuildRenewLeasesCommand(connection)) { TelemetryInstance.TrackEvent(TelemetryEventName.RenewLeasesStart, this._telemetryProps); + this._logger.LogDebugWithThreadId($"BEGIN RenewLeases Query={renewLeasesCommand.CommandText}"); var stopwatch = Stopwatch.StartNew(); await renewLeasesCommand.ExecuteNonQueryAsync(token); + long durationMs = stopwatch.ElapsedMilliseconds; + this._logger.LogDebugWithThreadId($"END RenewLeases Duration={durationMs}ms"); var measures = new Dictionary { - [TelemetryMeasureName.DurationMs] = stopwatch.ElapsedMilliseconds, + [TelemetryMeasureName.DurationMs] = durationMs, }; TelemetryInstance.TrackEvent(TelemetryEventName.RenewLeasesEnd, this._telemetryProps, measures); @@ -437,6 +454,7 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken } // Want to always release the lock at the end, even if renewing the leases failed. + this._logger.LogDebugWithThreadId("ReleaseRowLock - RenewLeases"); this._rowsLock.Release(); } } @@ -449,12 +467,15 @@ private async Task ClearRowsAsync(bool acquireLock) { if (acquireLock) { + this._logger.LogDebugWithThreadId("BEGIN WaitRowsLock - ClearRows"); await this._rowsLock.WaitAsync(); + this._logger.LogDebugWithThreadId("END WaitRowsLock - ClearRows"); } this._leaseRenewalCount = 0; this._state = State.CheckingForChanges; this._rows = new List>(); + this._logger.LogDebugWithThreadId("ReleaseRowLock - ClearRows"); this._rowsLock.Release(); } @@ -465,9 +486,10 @@ private async Task ClearRowsAsync(bool acquireLock) private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToken token) { TelemetryInstance.TrackEvent(TelemetryEventName.ReleaseLeasesStart, this._telemetryProps); - + this._logger.LogDebugWithThreadId("BEGIN WaitRowsLock - ReleaseLeases"); // Don't want to change the "_rows" while another thread is attempting to renew leases on them. await this._rowsLock.WaitAsync(token); + this._logger.LogDebugWithThreadId("END WaitRowsLock - ReleaseLeases"); long newLastSyncVersion = this.RecomputeLastSyncVersion(); try @@ -482,18 +504,22 @@ private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToke // Release the leases held on "_rows". using (SqlCommand releaseLeasesCommand = this.BuildReleaseLeasesCommand(connection, transaction)) { + this._logger.LogDebugWithThreadId($"BEGIN ReleaseLeases Query={releaseLeasesCommand.CommandText}"); var commandSw = Stopwatch.StartNew(); await releaseLeasesCommand.ExecuteNonQueryAsync(token); releaseLeasesDurationMs = commandSw.ElapsedMilliseconds; + this._logger.LogDebugWithThreadId($"END ReleaseLeases Duration={releaseLeasesDurationMs}ms"); } // Update the global state table if we have processed all changes with ChangeVersion <= newLastSyncVersion, // and clean up the leases table to remove all rows with ChangeVersion <= newLastSyncVersion. using (SqlCommand updateTablesPostInvocationCommand = this.BuildUpdateTablesPostInvocation(connection, transaction, newLastSyncVersion)) { + this._logger.LogDebugWithThreadId($"BEGIN UpdateTablesPostInvocation Query={updateTablesPostInvocationCommand.CommandText}"); var commandSw = Stopwatch.StartNew(); await updateTablesPostInvocationCommand.ExecuteNonQueryAsync(token); updateLastSyncVersionDurationMs = commandSw.ElapsedMilliseconds; + this._logger.LogDebugWithThreadId($"END UpdateTablesPostInvocation Duration={updateLastSyncVersionDurationMs}ms"); } transaction.Commit(); @@ -550,10 +576,11 @@ private long RecomputeLastSyncVersion() string changeVersion = row["SYS_CHANGE_VERSION"]; changeVersionSet.Add(long.Parse(changeVersion, CultureInfo.InvariantCulture)); } - // If there are more than one version numbers in the set, return the second highest one. Otherwise, return // the only version number in the set. - return changeVersionSet.ElementAt(changeVersionSet.Count > 1 ? changeVersionSet.Count - 2 : 0); + long lastSyncVersion = changeVersionSet.ElementAt(changeVersionSet.Count > 1 ? changeVersionSet.Count - 2 : 0); + this._logger.LogDebugWithThreadId($"RecomputeLastSyncVersion. LastSyncVersion={lastSyncVersion} ChangeVersionSet={string.Join(",", changeVersionSet)}"); + return lastSyncVersion; } /// @@ -562,8 +589,9 @@ private long RecomputeLastSyncVersion() /// will be populated with only the primary key values of the deleted row. /// /// The list of changes - private IReadOnlyList> GetChanges() + private IReadOnlyList> ProcessChanges() { + this._logger.LogDebugWithThreadId("BEGIN ProcessChanges"); var changes = new List>(); foreach (IReadOnlyDictionary row in this._rows) { @@ -577,7 +605,7 @@ private IReadOnlyList> GetChanges() changes.Add(new SqlChange(operation, JsonConvert.DeserializeObject(JsonConvert.SerializeObject(item)))); } - + this._logger.LogDebugWithThreadId("END ProcessChanges"); return changes; } @@ -616,7 +644,7 @@ private SqlCommand BuildUpdateTablesPreInvocation(SqlConnection connection, SqlT SELECT @last_sync_version = LastSyncVersion FROM {SqlTriggerConstants.GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; - + IF @last_sync_version < @min_valid_version UPDATE {SqlTriggerConstants.GlobalStateTableName} SET LastSyncVersion = @min_valid_version diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 46ac1e708..5523ffc69 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -76,9 +76,6 @@ public void Dispose() public async Task StartAsync(CancellationToken cancellationToken) { - this.InitializeTelemetryProps(); - TelemetryInstance.TrackEvent(TelemetryEventName.StartListenerStart, this._telemetryProps); - int previousState = Interlocked.CompareExchange(ref this._listenerState, ListenerStarting, ListenerNotStarted); switch (previousState) @@ -88,11 +85,16 @@ public async Task StartAsync(CancellationToken cancellationToken) default: break; } + this.InitializeTelemetryProps(); + TelemetryInstance.TrackEvent(TelemetryEventName.StartListenerStart, this._telemetryProps); + try { using (var connection = new SqlConnection(this._connectionString)) { + this._logger.LogDebugWithThreadId("BEGIN OpenListenerConnection"); await connection.OpenAsync(cancellationToken); + this._logger.LogDebugWithThreadId("END OpenListenerConnection"); this._telemetryProps.AddConnectionProps(connection); int userTableId = await this.GetUserTableIdAsync(connection, cancellationToken); @@ -100,7 +102,6 @@ public async Task StartAsync(CancellationToken cancellationToken) IReadOnlyList userTableColumns = await this.GetUserTableColumnsAsync(connection, userTableId, cancellationToken); string leasesTableName = string.Format(CultureInfo.InvariantCulture, SqlTriggerConstants.LeasesTableNameFormat, $"{this._userFunctionId}_{userTableId}"); - this._logger.LogDebug($"leases table name: '{leasesTableName}'."); this._telemetryProps[TelemetryPropertyName.LeasesTableName] = leasesTableName; var transactionSw = Stopwatch.StartNew(); @@ -108,10 +109,10 @@ public async Task StartAsync(CancellationToken cancellationToken) using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) { - createdSchemaDurationMs = await CreateSchemaAsync(connection, transaction, cancellationToken); - createGlobalStateTableDurationMs = await CreateGlobalStateTableAsync(connection, transaction, cancellationToken); + createdSchemaDurationMs = await this.CreateSchemaAsync(connection, transaction, cancellationToken); + createGlobalStateTableDurationMs = await this.CreateGlobalStateTableAsync(connection, transaction, cancellationToken); insertGlobalStateTableRowDurationMs = await this.InsertGlobalStateTableRowAsync(connection, transaction, userTableId, cancellationToken); - createLeasesTableDurationMs = await CreateLeasesTableAsync(connection, transaction, leasesTableName, primaryKeyColumns, cancellationToken); + createLeasesTableDurationMs = await this.CreateLeasesTableAsync(connection, transaction, leasesTableName, primaryKeyColumns, cancellationToken); transaction.Commit(); } @@ -186,6 +187,7 @@ private async Task GetUserTableIdAsync(SqlConnection connection, Cancellati { string getObjectIdQuery = $"SELECT OBJECT_ID(N{this._userTable.QuotedFullName}, 'U');"; + this._logger.LogDebugWithThreadId($"BEGIN GetUserTableId Query={getObjectIdQuery}"); using (var getObjectIdCommand = new SqlCommand(getObjectIdQuery, connection)) using (SqlDataReader reader = await getObjectIdCommand.ExecuteReaderAsync(cancellationToken)) { @@ -200,7 +202,7 @@ private async Task GetUserTableIdAsync(SqlConnection connection, Cancellati { throw new InvalidOperationException($"Could not find table: '{this._userTable.FullName}'."); } - + this._logger.LogDebugWithThreadId($"END GetUserTableId TableId={userTableId}"); return (int)userTableId; } } @@ -221,7 +223,7 @@ FROM sys.indexes AS i INNER JOIN sys.types AS t ON c.user_type_id = t.user_type_id WHERE i.is_primary_key = 1 AND i.object_id = {userTableId}; "; - + this._logger.LogDebugWithThreadId($"BEGIN GetPrimaryKeyColumns Query={getPrimaryKeyColumnsQuery}"); using (var getPrimaryKeyColumnsCommand = new SqlCommand(getPrimaryKeyColumnsQuery, connection)) using (SqlDataReader reader = await getPrimaryKeyColumnsCommand.ExecuteReaderAsync(cancellationToken)) { @@ -273,7 +275,7 @@ FROM sys.indexes AS i " Please rename them to be able to use trigger binding."); } - this._logger.LogDebug($"Primary key column names(types): {string.Join(", ", primaryKeyColumns.Select(col => $"'{col.name}({col.type})'"))}."); + this._logger.LogDebugWithThreadId($"END GetPrimaryKeyColumns ColumnNames(types) = {string.Join(", ", primaryKeyColumns.Select(col => $"'{col.name}({col.type})'"))}."); return primaryKeyColumns; } } @@ -290,6 +292,7 @@ FROM sys.columns AS c WHERE c.object_id = {userTableId}; "; + this._logger.LogDebugWithThreadId($"BEGIN GetUserTableColumns Query={getUserTableColumnsQuery}"); using (var getUserTableColumnsCommand = new SqlCommand(getUserTableColumnsQuery, connection)) using (SqlDataReader reader = await getUserTableColumnsCommand.ExecuteReaderAsync(cancellationToken)) { @@ -316,7 +319,7 @@ FROM sys.columns AS c throw new InvalidOperationException($"Found column(s) with unsupported type(s): {columnNamesAndTypes} in table: '{this._userTable.FullName}'."); } - this._logger.LogDebug($"User table column names: {string.Join(", ", userTableColumns.Select(col => $"'{col}'"))}."); + this._logger.LogDebugWithThreadId($"END GetUserTableColumns ColumnNames = {string.Join(", ", userTableColumns.Select(col => $"'{col}'"))}."); return userTableColumns; } } @@ -324,25 +327,36 @@ FROM sys.columns AS c /// /// Creates the schema for global state table and leases tables, if it does not already exist. /// - private static async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) + /// The already-opened connection to use for executing the command + /// The transaction wrapping this command + /// Cancellation token to pass to the command + /// The time taken in ms to execute the command + private async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { string createSchemaQuery = $@" IF SCHEMA_ID(N'{SqlTriggerConstants.SchemaName}') IS NULL EXEC ('CREATE SCHEMA {SqlTriggerConstants.SchemaName}'); "; + this._logger.LogDebugWithThreadId($"BEGIN CreateSchema Query={createSchemaQuery}"); using (var createSchemaCommand = new SqlCommand(createSchemaQuery, connection, transaction)) { var stopwatch = Stopwatch.StartNew(); await createSchemaCommand.ExecuteNonQueryAsync(cancellationToken); - return stopwatch.ElapsedMilliseconds; + long durationMs = stopwatch.ElapsedMilliseconds; + this._logger.LogDebugWithThreadId($"END CreateSchema Duration={durationMs}ms"); + return durationMs; } } /// /// Creates the global state table if it does not already exist. /// - private static async Task CreateGlobalStateTableAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) + /// The already-opened connection to use for executing the command + /// The transaction wrapping this command + /// Cancellation token to pass to the command + /// The time taken in ms to execute the command + private async Task CreateGlobalStateTableAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { string createGlobalStateTableQuery = $@" IF OBJECT_ID(N'{SqlTriggerConstants.GlobalStateTableName}', 'U') IS NULL @@ -354,23 +368,32 @@ PRIMARY KEY (UserFunctionID, UserTableID) ); "; + this._logger.LogDebugWithThreadId($"BEGIN CreateGlobalStateTable Query={createGlobalStateTableQuery}"); using (var createGlobalStateTableCommand = new SqlCommand(createGlobalStateTableQuery, connection, transaction)) { var stopwatch = Stopwatch.StartNew(); await createGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); - return stopwatch.ElapsedMilliseconds; + long durationMs = stopwatch.ElapsedMilliseconds; + this._logger.LogDebugWithThreadId($"END CreateGlobalStateTable Duration={durationMs}ms"); + return durationMs; } } /// /// Inserts row for the 'user function and table' inside the global state table, if one does not already exist. /// + /// The already-opened connection to use for executing the command + /// The transaction wrapping this command + /// Cancellation token to pass to the command + /// The time taken in ms to execute the command private async Task InsertGlobalStateTableRowAsync(SqlConnection connection, SqlTransaction transaction, int userTableId, CancellationToken cancellationToken) { object minValidVersion; string getMinValidVersionQuery = $"SELECT CHANGE_TRACKING_MIN_VALID_VERSION({userTableId});"; + this._logger.LogDebugWithThreadId($"BEGIN InsertGlobalStateTableRow"); + this._logger.LogDebugWithThreadId($"BEGIN GetMinValidVersion Query={getMinValidVersionQuery}"); using (var getMinValidVersionCommand = new SqlCommand(getMinValidVersionQuery, connection, transaction)) using (SqlDataReader reader = await getMinValidVersionCommand.ExecuteReaderAsync(cancellationToken)) { @@ -386,6 +409,7 @@ private async Task InsertGlobalStateTableRowAsync(SqlConnection connection throw new InvalidOperationException($"Could not find change tracking enabled for table: '{this._userTable.FullName}'."); } } + this._logger.LogDebugWithThreadId($"END GetMinValidVersion MinValidVersion={minValidVersion}"); string insertRowGlobalStateTableQuery = $@" IF NOT EXISTS ( @@ -396,18 +420,28 @@ INSERT INTO {SqlTriggerConstants.GlobalStateTableName} VALUES ('{this._userFunctionId}', {userTableId}, {(long)minValidVersion}); "; + this._logger.LogDebugWithThreadId($"BEGIN InsertRowGlobalStateTableQuery Query={insertRowGlobalStateTableQuery}"); using (var insertRowGlobalStateTableCommand = new SqlCommand(insertRowGlobalStateTableQuery, connection, transaction)) { var stopwatch = Stopwatch.StartNew(); await insertRowGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); - return stopwatch.ElapsedMilliseconds; + long durationMs = stopwatch.ElapsedMilliseconds; + this._logger.LogDebugWithThreadId($"END InsertRowGlobalStateTableQuery Duration={durationMs}ms"); + this._logger.LogDebugWithThreadId("END InsertGlobalStateTableRow"); + return durationMs; } } /// /// Creates the leases table for the 'user function and table', if one does not already exist. /// - private static async Task CreateLeasesTableAsync( + /// The already-opened connection to use for executing the command + /// The transaction wrapping this command + /// The name of the leases table to create + /// The primary keys of the user table this leases table is for + /// Cancellation token to pass to the command + /// The time taken in ms to execute the command + private async Task CreateLeasesTableAsync( SqlConnection connection, SqlTransaction transaction, string leasesTableName, @@ -428,11 +462,14 @@ PRIMARY KEY ({primaryKeys}) ); "; + this._logger.LogDebugWithThreadId($"BEGIN CreateLeasesTable Query={createLeasesTableQuery}"); using (var createLeasesTableCommand = new SqlCommand(createLeasesTableQuery, connection, transaction)) { var stopwatch = Stopwatch.StartNew(); await createLeasesTableCommand.ExecuteNonQueryAsync(cancellationToken); - return stopwatch.ElapsedMilliseconds; + long durationMs = stopwatch.ElapsedMilliseconds; + this._logger.LogDebugWithThreadId($"END CreateLeasesTable Duration={durationMs}ms"); + return durationMs; } } diff --git a/src/Utils.cs b/src/Utils.cs index 240ba8645..d05d01a9a 100644 --- a/src/Utils.cs +++ b/src/Utils.cs @@ -4,6 +4,7 @@ using System; using System.Linq; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using MoreLinq; using Newtonsoft.Json.Linq; @@ -93,5 +94,10 @@ public static void LowercasePropertyNames(this JObject obj) property.Replace(new JProperty(property.Name.ToLowerInvariant(), property.Value)); } } + + public static void LogDebugWithThreadId(this ILogger logger, string message, params object[] args) + { + logger.LogDebug($"TID:{Environment.CurrentManagedThreadId} {message}", args); + } } } From f912d2cfbacf115be251e529ef5de252014b1541 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Fri, 9 Sep 2022 05:54:43 +0530 Subject: [PATCH 21/77] Do not process rows if lease acquisition fails (#339) --- src/TriggerBinding/SqlTableChangeMonitor.cs | 35 ++++++++++++--------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index d0f8241f0..e72b50535 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -217,12 +217,13 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo } this._logger.LogDebugWithThreadId($"END UpdateTablesPreInvocation Duration={setLastSyncVersionDurationMs}ms"); + var rows = new List>(); + // Use the version number to query for new changes. using (SqlCommand getChangesCommand = this.BuildGetChangesCommand(connection, transaction)) { this._logger.LogDebugWithThreadId($"BEGIN GetChanges Query={getChangesCommand.CommandText}"); var commandSw = Stopwatch.StartNew(); - var rows = new List>(); using (SqlDataReader reader = await getChangesCommand.ExecuteReaderAsync(token)) { @@ -232,15 +233,14 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo } } - this._rows = rows; getChangesDurationMs = commandSw.ElapsedMilliseconds; } - this._logger.LogDebugWithThreadId($"END GetChanges Duration={getChangesDurationMs}ms ChangedRows={this._rows.Count}"); + this._logger.LogDebugWithThreadId($"END GetChanges Duration={getChangesDurationMs}ms ChangedRows={rows.Count}"); // If changes were found, acquire leases on them. - if (this._rows.Count > 0) + if (rows.Count > 0) { - using (SqlCommand acquireLeasesCommand = this.BuildAcquireLeasesCommand(connection, transaction)) + using (SqlCommand acquireLeasesCommand = this.BuildAcquireLeasesCommand(connection, transaction, rows)) { this._logger.LogDebugWithThreadId($"BEGIN AcquireLeases Query={acquireLeasesCommand.CommandText}"); var commandSw = Stopwatch.StartNew(); @@ -252,6 +252,9 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo transaction.Commit(); + // Set the rows for processing, now since the leases are acquired. + this._rows = rows; + var measures = new Dictionary { [TelemetryMeasureName.SetLastSyncVersionDurationMs] = setLastSyncVersionDurationMs, @@ -576,6 +579,7 @@ private long RecomputeLastSyncVersion() string changeVersion = row["SYS_CHANGE_VERSION"]; changeVersionSet.Add(long.Parse(changeVersion, CultureInfo.InvariantCulture)); } + // If there are more than one version numbers in the set, return the second highest one. Otherwise, return // the only version number in the set. long lastSyncVersion = changeVersionSet.ElementAt(changeVersionSet.Count > 1 ? changeVersionSet.Count - 2 : 0); @@ -696,15 +700,16 @@ LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCo /// /// The connection to add to the returned SqlCommand /// The transaction to add to the returned SqlCommand + /// Dictionary representing the table rows on which leases should be acquired /// The SqlCommand populated with the query and appropriate parameters - private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransaction transaction) + private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransaction transaction, IReadOnlyList> rows) { var acquireLeasesQuery = new StringBuilder(); - for (int rowIndex = 0; rowIndex < this._rows.Count; rowIndex++) + for (int rowIndex = 0; rowIndex < rows.Count; rowIndex++) { string valuesList = string.Join(", ", this._primaryKeyColumns.Select((_, colIndex) => $"@{rowIndex}_{colIndex}")); - string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"]; + string changeVersion = rows[rowIndex]["SYS_CHANGE_VERSION"]; acquireLeasesQuery.Append($@" IF NOT EXISTS (SELECT * FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}) @@ -720,7 +725,7 @@ IF NOT EXISTS (SELECT * FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {this "); } - return this.GetSqlCommandWithParameters(acquireLeasesQuery.ToString(), connection, transaction); + return this.GetSqlCommandWithParameters(acquireLeasesQuery.ToString(), connection, transaction, rows); } /// @@ -738,7 +743,7 @@ private SqlCommand BuildRenewLeasesCommand(SqlConnection connection) WHERE {matchCondition}; "; - return this.GetSqlCommandWithParameters(renewLeasesQuery, connection, null); + return this.GetSqlCommandWithParameters(renewLeasesQuery, connection, null, this._rows); } /// @@ -771,7 +776,7 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa "); } - return this.GetSqlCommandWithParameters(releaseLeasesQuery.ToString(), connection, transaction); + return this.GetSqlCommandWithParameters(releaseLeasesQuery.ToString(), connection, transaction, this._rows); } /// @@ -827,6 +832,7 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_ /// SQL query string /// The connection to add to the returned SqlCommand /// The transaction to add to the returned SqlCommand + /// Dictionary representing the table rows /// /// Ideally, we would have a map that maps from rows to a list of SqlCommands populated with their primary key /// values. The issue with this is that SQL doesn't seem to allow adding parameters to one collection when they @@ -834,12 +840,13 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_ /// is thrown if they are also added to the collection of a SqlCommand. The expected behavior seems to be to /// rebuild the SqlParameters each time. /// - private SqlCommand GetSqlCommandWithParameters(string commandText, SqlConnection connection, SqlTransaction transaction) + private SqlCommand GetSqlCommandWithParameters(string commandText, SqlConnection connection, + SqlTransaction transaction, IReadOnlyList> rows) { var command = new SqlCommand(commandText, connection, transaction); - SqlParameter[] parameters = Enumerable.Range(0, this._rows.Count) - .SelectMany(rowIndex => this._primaryKeyColumns.Select((col, colIndex) => new SqlParameter($"@{rowIndex}_{colIndex}", this._rows[rowIndex][col]))) + SqlParameter[] parameters = Enumerable.Range(0, rows.Count) + .SelectMany(rowIndex => this._primaryKeyColumns.Select((col, colIndex) => new SqlParameter($"@{rowIndex}_{colIndex}", rows[rowIndex][col]))) .ToArray(); command.Parameters.AddRange(parameters); From a27b080a8c74cc5d33b4e6357473bbaa4430e837 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Wed, 14 Sep 2022 07:28:41 +0530 Subject: [PATCH 22/77] Wait for error inside `StartFunctionsHostAndWaitForError` method (#349) --- test/Integration/SqlTriggerBindingIntegrationTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index c92ab7a83..cca96e883 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -269,9 +269,15 @@ void OutputHandler(object sender, DataReceivedEventArgs e) // All trigger integration tests are only using C# functions for testing at the moment. this.StartFunctionHost(functionName, Common.SupportedLanguages.CSharp, useTestFolder, OutputHandler); + + // The functions host generally logs the error message within a second after starting up. + const int BufferTimeForErrorInSeconds = 15; + bool isCompleted = tcs.Task.Wait(TimeSpan.FromSeconds(BufferTimeForErrorInSeconds)); + this.FunctionHost.OutputDataReceived -= OutputHandler; this.FunctionHost.Kill(); + Assert.True(isCompleted, "Functions host did not log failure to start SQL trigger listener within specified time."); Assert.Equal(expectedErrorMessage, errorMessage); } } From f23083b7ce12f2d117c6734ee40746fc029a2457 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Wed, 14 Sep 2022 07:30:45 +0530 Subject: [PATCH 23/77] Add test for multiple triggered functions (#343) --- .../SqlTriggerBindingIntegrationTests.cs | 60 ++++++++++++++++--- .../test-csharp/MultiFunctionTrigger.cs | 34 +++++++++++ 2 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 test/Integration/test-csharp/MultiFunctionTrigger.cs diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index cca96e883..ccea60489 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -23,8 +23,7 @@ public SqlTriggerBindingIntegrationTests(ITestOutputHelper output) : base(output } /// - /// Ensures that the user function gets invoked for each of the insert, update and delete operation, and the - /// changes to the user table are passed to the function in correct sequence. + /// Ensures that the user function gets invoked for each of the insert, update and delete operation. /// [Fact] public async Task SingleOperationTriggerTest() @@ -33,7 +32,7 @@ public async Task SingleOperationTriggerTest() this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); var changes = new List>(); - this.MonitorProductChanges(changes); + this.MonitorProductChanges(changes, "SQL Changes: "); // Considering the polling interval of 5 seconds and batch-size of 10, it should take around 15 seconds to // process 30 insert operations. Similar reasoning is used to set delays for update and delete operations. @@ -68,7 +67,7 @@ public async Task MultiOperationTriggerTest() this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); var changes = new List>(); - this.MonitorProductChanges(changes); + this.MonitorProductChanges(changes, "SQL Changes: "); // Insert + multiple updates to a row are treated as single insert with latest row values. this.InsertProducts(1, 5); @@ -97,6 +96,52 @@ public async Task MultiOperationTriggerTest() changes.Clear(); } + + /// + /// Ensures correct functionality with multiple user functions tracking the same table. + /// + [Fact] + public async Task MultiFunctionTriggerTest() + { + this.EnableChangeTrackingForTable("Products"); + + string functionList = $"{nameof(MultiFunctionTrigger.MultiFunctionTrigger1)} {nameof(MultiFunctionTrigger.MultiFunctionTrigger2)}"; + this.StartFunctionHost(functionList, Common.SupportedLanguages.CSharp, useTestFolder: true); + + var changes1 = new List>(); + var changes2 = new List>(); + + this.MonitorProductChanges(changes1, "Trigger1 Changes: "); + this.MonitorProductChanges(changes2, "Trigger2 Changes: "); + + // Considering the polling interval of 5 seconds and batch-size of 10, it should take around 15 seconds to + // process 30 insert operations for each trigger-listener. Similar reasoning is used to set delays for + // update and delete operations. + this.InsertProducts(1, 30); + await Task.Delay(TimeSpan.FromSeconds(20)); + ValidateProductChanges(changes1, 1, 30, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); + ValidateProductChanges(changes2, 1, 30, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); + changes1.Clear(); + changes2.Clear(); + + // All table columns (not just the columns that were updated) would be returned for update operation. + this.UpdateProducts(1, 20); + await Task.Delay(TimeSpan.FromSeconds(15)); + ValidateProductChanges(changes1, 1, 20, SqlChangeOperation.Update, id => $"Updated Product {id}", id => id * 100); + ValidateProductChanges(changes2, 1, 20, SqlChangeOperation.Update, id => $"Updated Product {id}", id => id * 100); + changes1.Clear(); + changes2.Clear(); + + // The properties corresponding to non-primary key columns would be set to the C# type's default values + // (null and 0) for delete operation. + this.DeleteProducts(11, 30); + await Task.Delay(TimeSpan.FromSeconds(15)); + ValidateProductChanges(changes1, 11, 30, SqlChangeOperation.Delete, _ => null, _ => 0); + ValidateProductChanges(changes2, 11, 30, SqlChangeOperation.Delete, _ => null, _ => 0); + changes1.Clear(); + changes2.Clear(); + } + /// /// Tests the error message when the user table is not present in the database. /// @@ -177,16 +222,15 @@ ALTER TABLE [dbo].[{tableName}] "); } - private void MonitorProductChanges(List> changes) + private void MonitorProductChanges(List> changes, string messagePrefix) { int index = 0; - string prefix = "SQL Changes: "; this.FunctionHost.OutputDataReceived += (sender, e) => { - if (e.Data != null && (index = e.Data.IndexOf(prefix, StringComparison.Ordinal)) >= 0) + if (e.Data != null && (index = e.Data.IndexOf(messagePrefix, StringComparison.Ordinal)) >= 0) { - string json = e.Data[(index + prefix.Length)..]; + string json = e.Data[(index + messagePrefix.Length)..]; changes.AddRange(JsonConvert.DeserializeObject>>(json)); } }; diff --git a/test/Integration/test-csharp/MultiFunctionTrigger.cs b/test/Integration/test-csharp/MultiFunctionTrigger.cs new file mode 100644 index 000000000..66d25b267 --- /dev/null +++ b/test/Integration/test-csharp/MultiFunctionTrigger.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration +{ + /// + /// Used to ensure correct functionality with multiple user functions tracking the same table. + /// + public static class MultiFunctionTrigger + { + [FunctionName(nameof(MultiFunctionTrigger1))] + public static void MultiFunctionTrigger1( + [SqlTrigger("[dbo].[Products]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> products, + ILogger logger) + { + logger.LogInformation("Trigger1 Changes: " + JsonConvert.SerializeObject(products)); + } + + [FunctionName(nameof(MultiFunctionTrigger2))] + public static void MultiFunctionTrigger2( + [SqlTrigger("[dbo].[Products]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> products, + ILogger logger) + { + logger.LogInformation("Trigger2 Changes: " + JsonConvert.SerializeObject(products)); + } + } +} From f42d2738675ad69756ed63ab309849bbe9355438 Mon Sep 17 00:00:00 2001 From: AmeyaRele <35621237+AmeyaRele@users.noreply.github.com> Date: Wed, 14 Sep 2022 10:54:48 +0530 Subject: [PATCH 24/77] Add retry when attempting to release leases (#340) * Add retry when attempting to release leases * Address comments * Address more comments * Change loop structure according to comment --- src/Telemetry/Telemetry.cs | 2 + src/TriggerBinding/SqlTableChangeMonitor.cs | 42 ++++++++++++--------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index 75853419b..2b3f9b8de 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -384,6 +384,7 @@ public enum TelemetryMeasureName GetPrimaryKeysDurationMs, InsertGlobalStateTableRowDurationMs, ReleaseLeasesDurationMs, + RetryAttemptNumber, SetLastSyncVersionDurationMs, TransactionDurationMs, UpdateLastSyncVersionDurationMs, @@ -408,6 +409,7 @@ public enum TelemetryErrorName ProcessChanges, PropsNotExistOnTable, ReleaseLeases, + ReleaseLeasesNoRetriesLeft, ReleaseLeasesRollback, RenewLeases, RenewLeasesLoop, diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index e72b50535..908f10248 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -35,7 +35,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable public const int MaxLeaseRenewalCount = 10; public const int LeaseIntervalInSeconds = 60; public const int LeaseRenewalIntervalInSeconds = 15; - + public const int MaxRetryReleaseLeases = 3; private readonly string _connectionString; private readonly int _userTableId; private readonly SqlObject _userTable; @@ -494,8 +494,9 @@ private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToke await this._rowsLock.WaitAsync(token); this._logger.LogDebugWithThreadId("END WaitRowsLock - ReleaseLeases"); long newLastSyncVersion = this.RecomputeLastSyncVersion(); + bool retrySucceeded = false; - try + for (int retryCount = 1; retryCount <= MaxRetryReleaseLeases && !retrySucceeded; retryCount++) { var transactionSw = Stopwatch.StartNew(); long releaseLeasesDurationMs = 0L, updateLastSyncVersionDurationMs = 0L; @@ -531,14 +532,30 @@ private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToke { [TelemetryMeasureName.ReleaseLeasesDurationMs] = releaseLeasesDurationMs, [TelemetryMeasureName.UpdateLastSyncVersionDurationMs] = updateLastSyncVersionDurationMs, + [TelemetryMeasureName.TransactionDurationMs] = transactionSw.ElapsedMilliseconds, }; TelemetryInstance.TrackEvent(TelemetryEventName.ReleaseLeasesEnd, this._telemetryProps, measures); + retrySucceeded = true; } catch (Exception ex) { - this._logger.LogError($"Failed to execute SQL commands to release leases for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); - TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeases, ex, this._telemetryProps); + if (retryCount < MaxRetryReleaseLeases) + { + this._logger.LogError($"Failed to execute SQL commands to release leases in attempt: {retryCount} for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + + var measures = new Dictionary + { + [TelemetryMeasureName.RetryAttemptNumber] = retryCount, + }; + + TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeases, ex, this._telemetryProps, measures); + } + else + { + this._logger.LogError($"Failed to release leases for table '{this._userTable.FullName}' after {MaxRetryReleaseLeases} attempts due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeasesNoRetriesLeft, ex, this._telemetryProps); + } try { @@ -552,20 +569,9 @@ private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToke } } } - catch (Exception e) - { - // What should we do if releasing the leases fails? We could try to release them again or just wait, - // since eventually the lease time will expire. Then another thread will re-process the same changes - // though, so less than ideal. But for now that's the functionality. - this._logger.LogError($"Failed to release leases for table '{this._userTable.FullName}' due to exception: {e.GetType()}. Exception message: {e.Message}"); - TelemetryInstance.TrackException(TelemetryErrorName.ReleaseLeases, e, this._telemetryProps); - } - finally - { - // Want to do this before releasing the lock in case the renew leases thread wakes up. It will see that - // the state is checking for changes and not renew the (just released) leases. - await this.ClearRowsAsync(false); - } + // Want to do this before releasing the lock in case the renew leases thread wakes up. It will see that + // the state is checking for changes and not renew the (just released) leases. + await this.ClearRowsAsync(false); } /// From 1dad068c24c778342bf146d5e4632bbacfe6d1ff Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Fri, 16 Sep 2022 07:05:07 +0530 Subject: [PATCH 25/77] Refactor locking logic in class `SqlTableChangeMonitor` (#357) --- src/TriggerBinding/SqlTableChangeMonitor.cs | 82 ++++++++++----------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 908f10248..08f6f60ee 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -51,14 +51,18 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly CancellationTokenSource _cancellationTokenSourceRenewLeases; private CancellationTokenSource _cancellationTokenSourceExecutor; - // The semaphore ensures that mutable class members such as this._rows are accessed by only one thread at a time. + // The semaphore gets used by lease-renewal loop to ensure that '_state' stays set to 'ProcessingChanges' while + // the leases are being renewed. The change-consumption loop requires to wait for the semaphore before modifying + // the value of '_state' back to 'CheckingForChanges'. Since the field '_rows' is only updated if the value of + // '_state' is set to 'CheckingForChanges', this guarantees that '_rows' will stay same while it is being + // iterated over inside the lease-renewal loop. private readonly SemaphoreSlim _rowsLock; private readonly IDictionary _telemetryProps; private IReadOnlyList> _rows; private int _leaseRenewalCount; - private State _state = State.CheckingForChanges; + private State _state; /// /// Initializes a new instance of the > class. @@ -116,7 +120,7 @@ public SqlTableChangeMonitor( this._telemetryProps = telemetryProps; - this._rowsLock = new SemaphoreSlim(1); + this._rowsLock = new SemaphoreSlim(1, 1); this._rows = new List>(); this._leaseRenewalCount = 0; this._state = State.CheckingForChanges; @@ -156,6 +160,7 @@ private async Task RunChangeConsumptionLoopAsync() this._logger.LogDebugWithThreadId("BEGIN OpenChangeConsumptionConnection"); await connection.OpenAsync(token); this._logger.LogDebugWithThreadId("END OpenChangeConsumptionConnection"); + // Check for cancellation request only after a cycle of checking and processing of changes completes. while (!token.IsCancellationRequested) { @@ -314,7 +319,7 @@ private async Task ProcessTableChangesAsync(SqlConnection connection, Cancellati { this._logger.LogError($"Failed to compose trigger parameter value for table: '{this._userTable.FullName} due to exception: {e.GetType()}. Exception message: {e.Message}"); TelemetryInstance.TrackException(TelemetryErrorName.ProcessChanges, e, this._telemetryProps); - await this.ClearRowsAsync(true); + await this.ClearRowsAsync(); } if (changes != null) @@ -346,7 +351,7 @@ private async Task ProcessTableChangesAsync(SqlConnection connection, Cancellati this._logger.LogError($"Failed to trigger user function for table: '{this._userTable.FullName} due to exception: {result.Exception.GetType()}. Exception message: {result.Exception.Message}"); TelemetryInstance.TrackException(TelemetryErrorName.ProcessChanges, result.Exception, this._telemetryProps, measures); - await this.ClearRowsAsync(true); + await this.ClearRowsAsync(); } } } @@ -370,11 +375,9 @@ private async void RunLeaseRenewalLoopAsync() this._logger.LogDebugWithThreadId("BEGIN OpenLeaseRenewalLoopConnection"); await connection.OpenAsync(token); this._logger.LogDebugWithThreadId("END OpenLeaseRenewalLoopConnection"); + while (!token.IsCancellationRequested) { - this._logger.LogDebugWithThreadId("BEGIN WaitRowsLock - LeaseRenewal"); - await this._rowsLock.WaitAsync(token); - this._logger.LogDebugWithThreadId("END WaitRowsLock - LeaseRenewal"); await this.RenewLeasesAsync(connection, token); await Task.Delay(TimeSpan.FromSeconds(LeaseRenewalIntervalInSeconds), token); } @@ -398,9 +401,13 @@ private async void RunLeaseRenewalLoopAsync() private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken token) { - try + this._logger.LogDebugWithThreadId("BEGIN WaitRowsLock - RenewLeases"); + await this._rowsLock.WaitAsync(token); + this._logger.LogDebugWithThreadId("END WaitRowsLock - RenewLeases"); + + if (this._state == State.ProcessingChanges) { - if (this._state == State.ProcessingChanges) + try { // I don't think I need a transaction for renewing leases. If this worker reads in a row from the // leases table and determines that it corresponds to its batch of changes, but then that row gets @@ -423,19 +430,16 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken TelemetryInstance.TrackEvent(TelemetryEventName.RenewLeasesEnd, this._telemetryProps, measures); } } - } - catch (Exception e) - { - // This catch block is necessary so that the finally block is executed even in the case of an exception - // (see https://docs.microsoft.com/dotnet/csharp/language-reference/keywords/try-finally, third - // paragraph). If we fail to renew the leases, multiple workers could be processing the same change - // data, but we have functionality in place to deal with this (see design doc). - this._logger.LogError($"Failed to renew leases due to exception: {e.GetType()}. Exception message: {e.Message}"); - TelemetryInstance.TrackException(TelemetryErrorName.RenewLeases, e, this._telemetryProps); - } - finally - { - if (this._state == State.ProcessingChanges) + catch (Exception e) + { + // This catch block is necessary so that the finally block is executed even in the case of an exception + // (see https://docs.microsoft.com/dotnet/csharp/language-reference/keywords/try-finally, third + // paragraph). If we fail to renew the leases, multiple workers could be processing the same change + // data, but we have functionality in place to deal with this (see design doc). + this._logger.LogError($"Failed to renew leases due to exception: {e.GetType()}. Exception message: {e.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.RenewLeases, e, this._telemetryProps); + } + finally { // Do we want to update this count even in the case of a failure to renew the leases? Probably, // because the count is simply meant to indicate how much time the other thread has spent processing @@ -455,30 +459,27 @@ private async Task RenewLeasesAsync(SqlConnection connection, CancellationToken this._cancellationTokenSourceExecutor = new CancellationTokenSource(); } } - - // Want to always release the lock at the end, even if renewing the leases failed. - this._logger.LogDebugWithThreadId("ReleaseRowLock - RenewLeases"); - this._rowsLock.Release(); } + + // Want to always release the lock at the end, even if renewing the leases failed. + this._logger.LogDebugWithThreadId("ReleaseRowsLock - RenewLeases"); + this._rowsLock.Release(); } /// /// Resets the in-memory state of the change monitor and sets it to start polling for changes again. /// - /// True if ClearRowsAsync should acquire the "_rowsLock" (only true in the case of a failure) - private async Task ClearRowsAsync(bool acquireLock) + private async Task ClearRowsAsync() { - if (acquireLock) - { - this._logger.LogDebugWithThreadId("BEGIN WaitRowsLock - ClearRows"); - await this._rowsLock.WaitAsync(); - this._logger.LogDebugWithThreadId("END WaitRowsLock - ClearRows"); - } + this._logger.LogDebugWithThreadId("BEGIN WaitRowsLock - ClearRows"); + await this._rowsLock.WaitAsync(); + this._logger.LogDebugWithThreadId("END WaitRowsLock - ClearRows"); this._leaseRenewalCount = 0; this._state = State.CheckingForChanges; this._rows = new List>(); - this._logger.LogDebugWithThreadId("ReleaseRowLock - ClearRows"); + + this._logger.LogDebugWithThreadId("ReleaseRowsLock - ClearRows"); this._rowsLock.Release(); } @@ -489,10 +490,6 @@ private async Task ClearRowsAsync(bool acquireLock) private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToken token) { TelemetryInstance.TrackEvent(TelemetryEventName.ReleaseLeasesStart, this._telemetryProps); - this._logger.LogDebugWithThreadId("BEGIN WaitRowsLock - ReleaseLeases"); - // Don't want to change the "_rows" while another thread is attempting to renew leases on them. - await this._rowsLock.WaitAsync(token); - this._logger.LogDebugWithThreadId("END WaitRowsLock - ReleaseLeases"); long newLastSyncVersion = this.RecomputeLastSyncVersion(); bool retrySucceeded = false; @@ -569,9 +566,8 @@ private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToke } } } - // Want to do this before releasing the lock in case the renew leases thread wakes up. It will see that - // the state is checking for changes and not renew the (just released) leases. - await this.ClearRowsAsync(false); + + await this.ClearRowsAsync(); } /// From 930f190218de667255e6b2fc448688329a0b41f1 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 2 Sep 2022 13:48:02 -0700 Subject: [PATCH 26/77] cherry-pick c5a746f7e70de3ed050289d3ab2651a39667abda --- performance/packages.lock.json | 61 ++++++++++++----------- samples/samples-csharp/packages.lock.json | 14 +++--- src/SqlAsyncCollector.cs | 6 +-- test/packages.lock.json | 24 ++++----- 4 files changed, 53 insertions(+), 52 deletions(-) diff --git a/performance/packages.lock.json b/performance/packages.lock.json index 343cf446a..f72a67a28 100644 --- a/performance/packages.lock.json +++ b/performance/packages.lock.json @@ -1145,8 +1145,8 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==" + "resolved": "4.7.0", + "contentHash": "oJjw3uFuVDJiJNbCD8HB4a2p3NYLdt1fiT5OGsPLw+WTOuG0KpP4OXelMmmVKpClueMsit6xOlzy4wNKQFiBLg==" }, "System.Diagnostics.FileVersionInfo": { "type": "Transitive", @@ -2088,46 +2088,47 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "2.17.0", - "Microsoft.AspNetCore.Http": "2.1.22", - "Microsoft.Azure.WebJobs": "3.0.31", - "Microsoft.Data.SqlClient": "3.0.1", - "Newtonsoft.Json": "13.0.1", - "System.Runtime.Caching": "4.7.0", - "morelinq": "3.3.2" + "Microsoft.ApplicationInsights": "[2.14.0, )", + "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.Azure.WebJobs": "[3.0.31, )", + "Microsoft.Data.SqlClient": "[3.0.1, )", + "Newtonsoft.Json": "[11.0.2, )", + "System.Runtime.Caching": "[4.7.0, )", + "morelinq": "[3.3.2, )" } }, "microsoft.azure.webjobs.extensions.sql.samples": { "type": "Project", "dependencies": { - "Microsoft.AspNetCore.Http": "2.1.22", - "Microsoft.Azure.WebJobs.Extensions.Sql": "99.99.99", - "Microsoft.Azure.WebJobs.Extensions.Storage": "5.0.0", - "Microsoft.NET.Sdk.Functions": "3.1.1", - "Newtonsoft.Json": "13.0.1" + "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", + "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )", + "Microsoft.NET.Sdk.Functions": "[3.1.1, )", + "Newtonsoft.Json": "[11.0.2, )" } }, "microsoft.azure.webjobs.extensions.sql.tests": { "type": "Project", "dependencies": { - "Microsoft.AspNetCore.Http": "2.1.22", - "Microsoft.Azure.WebJobs.Extensions.Sql": "99.99.99", - "Microsoft.Azure.WebJobs.Extensions.Sql.Samples": "1.0.0", - "Microsoft.NET.Sdk.Functions": "3.1.1", - "Microsoft.NET.Test.Sdk": "17.0.0", - "Moq": "4.14.3", - "Newtonsoft.Json": "13.0.1", - "xunit": "2.4.0", - "xunit.runner.visualstudio": "2.4.0" + "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", + "Microsoft.Azure.WebJobs.Extensions.Sql.Samples": "[1.0.0, )", + "Microsoft.NET.Sdk.Functions": "[3.1.1, )", + "Microsoft.NET.Test.Sdk": "[17.0.0, )", + "Moq": "[4.14.3, )", + "Newtonsoft.Json": "[11.0.2, )", + "xunit": "[2.4.0, )", + "xunit.runner.visualstudio": "[2.4.0, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.17.0, )", - "resolved": "2.17.0", - "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", + "requested": "[2.14.0, )", + "resolved": "2.14.0", + "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "5.0.0" + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory": "4.5.4" } }, "Microsoft.AspNetCore.Http": { @@ -2244,9 +2245,9 @@ }, "Newtonsoft.Json": { "type": "CentralTransitive", - "requested": "[13.0.1, )", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + "requested": "[11.0.2, )", + "resolved": "11.0.2", + "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" }, "System.Runtime.Caching": { "type": "CentralTransitive", diff --git a/samples/samples-csharp/packages.lock.json b/samples/samples-csharp/packages.lock.json index c7f928a9c..a7ba791dc 100644 --- a/samples/samples-csharp/packages.lock.json +++ b/samples/samples-csharp/packages.lock.json @@ -1727,13 +1727,13 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "2.14.0", - "Microsoft.AspNetCore.Http": "2.1.22", - "Microsoft.Azure.WebJobs": "3.0.31", - "Microsoft.Data.SqlClient": "3.0.1", - "Newtonsoft.Json": "11.0.2", - "System.Runtime.Caching": "4.7.0", - "morelinq": "3.3.2" + "Microsoft.ApplicationInsights": "[2.14.0, )", + "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.Azure.WebJobs": "[3.0.31, )", + "Microsoft.Data.SqlClient": "[3.0.1, )", + "Newtonsoft.Json": "[11.0.2, )", + "System.Runtime.Caching": "[4.7.0, )", + "morelinq": "[3.3.2, )" } }, "Microsoft.ApplicationInsights": { diff --git a/src/SqlAsyncCollector.cs b/src/SqlAsyncCollector.cs index e4fb6459f..2b385449c 100644 --- a/src/SqlAsyncCollector.cs +++ b/src/SqlAsyncCollector.cs @@ -349,7 +349,7 @@ private static void GenerateDataQueryForMerge(TableInformation table, IEnumerabl public class TableInformation { - public IEnumerable PrimaryKeys { get; } + public IEnumerable PrimaryKeys { get; } /// /// All of the columns, along with their data types, for SQL to use to turn JSON into a table @@ -383,7 +383,7 @@ public class TableInformation /// public JsonSerializerSettings JsonSerializerSettings { get; } - public TableInformation(IEnumerable primaryKeys, IDictionary columns, StringComparer comparer, string query, bool hasIdentityColumnPrimaryKeys) + public TableInformation(IEnumerable primaryKeys, IDictionary columns, StringComparer comparer, string query, bool hasIdentityColumnPrimaryKeys) { this.PrimaryKeys = primaryKeys; this.Columns = columns; @@ -618,7 +618,7 @@ public static async Task RetrieveTableInformationAsync(SqlConn // Match SQL Primary Key column names to POCO field/property objects. Ensure none are missing. StringComparison comparison = caseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; - IEnumerable primaryKeyFields = typeof(T).GetMembers().Where(f => primaryKeys.Any(k => string.Equals(k.Name, f.Name, comparison))); + IEnumerable primaryKeyFields = typeof(T).GetProperties().Where(f => primaryKeys.Any(k => string.Equals(k.Name, f.Name, comparison))); IEnumerable primaryKeysFromObject = columnNames.Where(f => primaryKeys.Any(k => string.Equals(k.Name, f, comparison))); IEnumerable missingPrimaryKeysFromItem = primaryKeys .Where(k => !primaryKeysFromObject.Contains(k.Name, comparer)); diff --git a/test/packages.lock.json b/test/packages.lock.json index 65443dcee..8a77ec072 100644 --- a/test/packages.lock.json +++ b/test/packages.lock.json @@ -1930,23 +1930,23 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "2.14.0", - "Microsoft.AspNetCore.Http": "2.1.22", - "Microsoft.Azure.WebJobs": "3.0.31", - "Microsoft.Data.SqlClient": "3.0.1", - "Newtonsoft.Json": "11.0.2", - "System.Runtime.Caching": "4.7.0", - "morelinq": "3.3.2" + "Microsoft.ApplicationInsights": "[2.14.0, )", + "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.Azure.WebJobs": "[3.0.31, )", + "Microsoft.Data.SqlClient": "[3.0.1, )", + "Newtonsoft.Json": "[11.0.2, )", + "System.Runtime.Caching": "[4.7.0, )", + "morelinq": "[3.3.2, )" } }, "microsoft.azure.webjobs.extensions.sql.samples": { "type": "Project", "dependencies": { - "Microsoft.AspNetCore.Http": "2.1.22", - "Microsoft.Azure.WebJobs.Extensions.Sql": "99.99.99", - "Microsoft.Azure.WebJobs.Extensions.Storage": "5.0.0", - "Microsoft.NET.Sdk.Functions": "3.1.1", - "Newtonsoft.Json": "11.0.2" + "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", + "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )", + "Microsoft.NET.Sdk.Functions": "[3.1.1, )", + "Newtonsoft.Json": "[11.0.2, )" } }, "Microsoft.ApplicationInsights": { From 3410e23960a789310737a5c334ddbb4f126ec5f2 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Mon, 19 Sep 2022 16:15:59 -0700 Subject: [PATCH 27/77] Merge commit '76575f29e964f3487684b84e279444279296f5c2' into chgagnon/mergeFromMain1 # Conflicts: # src/SqlAsyncCollector.cs # test/GlobalSuppressions.cs --- src/SqlAsyncCollector.cs | 8 +++- src/SqlBindingUtilities.cs | 4 +- test/Common/ProductColumnTypes.cs | 16 +++++++ test/Database/Tables/ProductsColumnTypes.sql | 5 +++ test/GlobalSuppressions.cs | 2 + .../SqlInputBindingIntegrationTests.cs | 40 ++++++++++++++++++ .../SqlOutputBindingIntegrationTests.cs | 21 ++++++++++ .../test-csharp/AddProductColumnTypes.cs | 35 ++++++++++++++++ .../GetProductColumnTypesSerialization.cs | 40 ++++++++++++++++++ ...olumnTypesSerializationDifferentCulture.cs | 42 +++++++++++++++++++ .../AddProductColumnTypes/function.json | 27 ++++++++++++ .../test-js/AddProductColumnTypes/index.js | 18 ++++++++ .../function.json | 28 +++++++++++++ .../index.js | 11 +++++ ....Azure.WebJobs.Extensions.Sql.Tests.csproj | 19 ++++----- 15 files changed, 302 insertions(+), 14 deletions(-) create mode 100644 test/Common/ProductColumnTypes.cs create mode 100644 test/Database/Tables/ProductsColumnTypes.sql create mode 100644 test/Integration/test-csharp/AddProductColumnTypes.cs create mode 100644 test/Integration/test-csharp/GetProductColumnTypesSerialization.cs create mode 100644 test/Integration/test-csharp/GetProductColumnTypesSerializationDifferentCulture.cs create mode 100644 test/Integration/test-js/AddProductColumnTypes/function.json create mode 100644 test/Integration/test-js/AddProductColumnTypes/index.js create mode 100644 test/Integration/test-js/GetProductsColumnTypesSerialization/function.json create mode 100644 test/Integration/test-js/GetProductsColumnTypesSerialization/index.js diff --git a/src/SqlAsyncCollector.cs b/src/SqlAsyncCollector.cs index 2b385449c..05d2f77b9 100644 --- a/src/SqlAsyncCollector.cs +++ b/src/SqlAsyncCollector.cs @@ -349,6 +349,8 @@ private static void GenerateDataQueryForMerge(TableInformation table, IEnumerabl public class TableInformation { + private const string ISO_8061_DATETIME_FORMAT = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff"; + public IEnumerable PrimaryKeys { get; } /// @@ -391,9 +393,13 @@ public TableInformation(IEnumerable primaryKeys, IDictionary /// Used to determine the columns of the table as well as the next SQL row to process /// The built dictionary - public static IReadOnlyDictionary BuildDictionaryFromSqlRow(SqlDataReader reader) + public static IReadOnlyDictionary BuildDictionaryFromSqlRow(SqlDataReader reader) { - return Enumerable.Range(0, reader.FieldCount).ToDictionary(reader.GetName, i => reader.GetValue(i).ToString()); + return Enumerable.Range(0, reader.FieldCount).ToDictionary(reader.GetName, i => reader.GetValue(i)); } /// diff --git a/test/Common/ProductColumnTypes.cs b/test/Common/ProductColumnTypes.cs new file mode 100644 index 000000000..97e36eb0b --- /dev/null +++ b/test/Common/ProductColumnTypes.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common +{ + public class ProductColumnTypes + { + public int ProductID { get; set; } + + public DateTime Datetime { get; set; } + + public DateTime Datetime2 { get; set; } + } +} diff --git a/test/Database/Tables/ProductsColumnTypes.sql b/test/Database/Tables/ProductsColumnTypes.sql new file mode 100644 index 000000000..a552b9db1 --- /dev/null +++ b/test/Database/Tables/ProductsColumnTypes.sql @@ -0,0 +1,5 @@ +CREATE TABLE [ProductsColumnTypes] ( + [ProductId] [int] NOT NULL PRIMARY KEY, + [Datetime] [datetime], + [Datetime2] [datetime2] +) \ No newline at end of file diff --git a/test/GlobalSuppressions.cs b/test/GlobalSuppressions.cs index 56034261e..b0fa905ab 100644 --- a/test/GlobalSuppressions.cs +++ b/test/GlobalSuppressions.cs @@ -17,3 +17,5 @@ [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.ReservedPrimaryKeyColumnNamesTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.TableNotPresentTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.UnsupportedColumnTypesTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] +[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples.GetProductsColumnTypesSerializationDifferentCulture.Run(Microsoft.AspNetCore.Http.HttpRequest,System.Collections.Generic.IAsyncEnumerable{Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common.ProductColumnTypes},Microsoft.Extensions.Logging.ILogger)~System.Threading.Tasks.Task{Microsoft.AspNetCore.Mvc.IActionResult}")] +[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples.GetProductsColumnTypesSerialization.Run(Microsoft.AspNetCore.Http.HttpRequest,System.Collections.Generic.IAsyncEnumerable{Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common.ProductColumnTypes},Microsoft.Extensions.Logging.ILogger)~System.Threading.Tasks.Task{Microsoft.AspNetCore.Mvc.IActionResult}")] diff --git a/test/Integration/SqlInputBindingIntegrationTests.cs b/test/Integration/SqlInputBindingIntegrationTests.cs index 9dc6532b9..7b63908a1 100644 --- a/test/Integration/SqlInputBindingIntegrationTests.cs +++ b/test/Integration/SqlInputBindingIntegrationTests.cs @@ -130,5 +130,45 @@ public async void GetProductNamesViewTest(SupportedLanguages lang) Assert.Equal(expectedResponse, TestUtils.CleanJsonString(actualResponse), StringComparer.OrdinalIgnoreCase); } + + /// + /// Verifies that serializing an item with various data types works when the language is + /// set to a non-enUS language. + /// + [Theory] + [SqlInlineData()] + [UnsupportedLanguages(SupportedLanguages.JavaScript)] // Javascript doesn't have the concept of a runtime language used during serialization + public async void GetProductsColumnTypesSerializationDifferentCultureTest(SupportedLanguages lang) + { + this.StartFunctionHost(nameof(GetProductsColumnTypesSerializationDifferentCulture), lang, true); + + this.ExecuteNonQuery("INSERT INTO [dbo].[ProductsColumnTypes] VALUES (" + + "999, " + // ProductId + "GETDATE(), " + // Datetime field + "GETDATE())"); // Datetime2 field + + await this.SendInputRequest("getproducts-columntypesserializationdifferentculture"); + + // If we get here the test has succeeded - it'll throw an exception if serialization fails + } + + /// + /// Verifies that serializing an item with various data types works as expected + /// + [Theory] + [SqlInlineData()] + public async void GetProductsColumnTypesSerializationTest(SupportedLanguages lang) + { + this.StartFunctionHost(nameof(GetProductsColumnTypesSerialization), lang, true); + + this.ExecuteNonQuery("INSERT INTO [dbo].[ProductsColumnTypes] VALUES (" + + "999, " + // ProductId + "GETDATE(), " + // Datetime field + "GETDATE())"); // Datetime2 field + + await this.SendInputRequest("getproducts-columntypesserialization"); + + // If we get here the test has succeeded - it'll throw an exception if serialization fails + } } } diff --git a/test/Integration/SqlOutputBindingIntegrationTests.cs b/test/Integration/SqlOutputBindingIntegrationTests.cs index 8f2f693c7..5b2d17d63 100644 --- a/test/Integration/SqlOutputBindingIntegrationTests.cs +++ b/test/Integration/SqlOutputBindingIntegrationTests.cs @@ -99,6 +99,27 @@ public void AddProductArrayTest(SupportedLanguages lang) Assert.Equal(2, this.ExecuteScalar("SELECT ProductId FROM Products WHERE Cost = 12")); } + /// + /// Test compatability with converting various data types to their respective + /// SQL server types. + /// + /// The language to run the test against + [Theory] + [SqlInlineData()] + public void AddProductColumnTypesTest(SupportedLanguages lang) + { + this.StartFunctionHost(nameof(AddProductColumnTypes), lang, true); + + var queryParameters = new Dictionary() + { + { "productId", "999" } + }; + + this.SendOutputGetRequest("addproduct-columntypes", queryParameters).Wait(); + + // If we get here then the test is successful - an exception will be thrown if there were any problems + } + [Theory] [SqlInlineData()] [UnsupportedLanguages(SupportedLanguages.JavaScript)] // Collectors are only available in C# diff --git a/test/Integration/test-csharp/AddProductColumnTypes.cs b/test/Integration/test-csharp/AddProductColumnTypes.cs new file mode 100644 index 000000000..180290ab1 --- /dev/null +++ b/test/Integration/test-csharp/AddProductColumnTypes.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration +{ + public static class AddProductColumnTypes + { + /// + /// This function is used to test compatability with converting various data types to their respective + /// SQL server types. + /// + [FunctionName(nameof(AddProductColumnTypes))] + public static IActionResult Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "addproduct-columntypes")] HttpRequest req, + [Sql("dbo.ProductsColumnTypes", ConnectionStringSetting = "SqlConnectionString")] out ProductColumnTypes product) + { + product = new ProductColumnTypes() + { + ProductID = int.Parse(req.Query["productId"]), + Datetime = DateTime.UtcNow, + Datetime2 = DateTime.UtcNow + }; + + // Items were inserted successfully so return success, an exception would be thrown if there + // was any issues + return new OkObjectResult("Success!"); + } + } +} diff --git a/test/Integration/test-csharp/GetProductColumnTypesSerialization.cs b/test/Integration/test-csharp/GetProductColumnTypesSerialization.cs new file mode 100644 index 000000000..836c305ca --- /dev/null +++ b/test/Integration/test-csharp/GetProductColumnTypesSerialization.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples +{ + public static class GetProductsColumnTypesSerialization + { + /// + /// This function verifies that serializing an item with various data types + /// works as expected. + /// Note this uses IAsyncEnumerable because IEnumerable serializes the entire table directly, + /// instead of each item one by one (which is where issues can occur) + /// + [FunctionName(nameof(GetProductsColumnTypesSerialization))] + public static async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "getproducts-columntypesserialization")] + HttpRequest req, + [Sql("SELECT * FROM [dbo].[ProductsColumnTypes]", + CommandType = System.Data.CommandType.Text, + ConnectionStringSetting = "SqlConnectionString")] + IAsyncEnumerable products, + ILogger log) + { + await foreach (ProductColumnTypes item in products) + { + log.LogInformation(JsonSerializer.Serialize(item)); + } + return new OkObjectResult(products); + } + } +} diff --git a/test/Integration/test-csharp/GetProductColumnTypesSerializationDifferentCulture.cs b/test/Integration/test-csharp/GetProductColumnTypesSerializationDifferentCulture.cs new file mode 100644 index 000000000..9417ba46a --- /dev/null +++ b/test/Integration/test-csharp/GetProductColumnTypesSerializationDifferentCulture.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples +{ + public static class GetProductsColumnTypesSerializationDifferentCulture + { + /// + /// This function verifies that serializing an item with various data types + /// works when the language is set to a non-enUS language. + /// Note this uses IAsyncEnumerable because IEnumerable serializes the entire table directly, + /// instead of each item one by one (which is where issues can occur) + /// + [FunctionName(nameof(GetProductsColumnTypesSerializationDifferentCulture))] + public static async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "getproducts-columntypesserializationdifferentculture")] + HttpRequest req, + [Sql("SELECT * FROM [dbo].[ProductsColumnTypes]", + CommandType = System.Data.CommandType.Text, + ConnectionStringSetting = "SqlConnectionString")] + IAsyncEnumerable products, + ILogger log) + { + CultureInfo.CurrentCulture = new CultureInfo("it-IT", false); + await foreach (ProductColumnTypes item in products) + { + log.LogInformation(JsonSerializer.Serialize(item)); + } + return new OkObjectResult(products); + } + } +} diff --git a/test/Integration/test-js/AddProductColumnTypes/function.json b/test/Integration/test-js/AddProductColumnTypes/function.json new file mode 100644 index 000000000..3b69ac6ae --- /dev/null +++ b/test/Integration/test-js/AddProductColumnTypes/function.json @@ -0,0 +1,27 @@ +{ + "bindings": [ + { + "authLevel": "function", + "name": "req", + "direction": "in", + "type": "httpTrigger", + "methods": [ + "get" + ], + "route": "addproduct-columntypes" + }, + { + "name": "$return", + "type": "http", + "direction": "out" + }, + { + "name": "product", + "type": "sql", + "direction": "out", + "commandText": "[dbo].[ProductsColumnTypes]", + "connectionStringSetting": "SqlConnectionString" + } + ], + "disabled": false +} \ No newline at end of file diff --git a/test/Integration/test-js/AddProductColumnTypes/index.js b/test/Integration/test-js/AddProductColumnTypes/index.js new file mode 100644 index 000000000..8d83ac300 --- /dev/null +++ b/test/Integration/test-js/AddProductColumnTypes/index.js @@ -0,0 +1,18 @@ +/** + * This function is used to test compatability with converting various data types to their respective + * SQL server types. + */ +module.exports = async function (context, req) { + const product = { + "productId": req.query.productId, + "datetime": Date.now(), + "datetime2": Date.now() + }; + + context.bindings.product = JSON.stringify(product); + + return { + status: 201, + body: product + }; +} \ No newline at end of file diff --git a/test/Integration/test-js/GetProductsColumnTypesSerialization/function.json b/test/Integration/test-js/GetProductsColumnTypesSerialization/function.json new file mode 100644 index 000000000..0554a9f7d --- /dev/null +++ b/test/Integration/test-js/GetProductsColumnTypesSerialization/function.json @@ -0,0 +1,28 @@ +{ + "bindings": [ + { + "authLevel": "function", + "name": "req", + "type": "httpTrigger", + "direction": "in", + "methods": [ + "get" + ], + "route": "getproducts-columntypesserialization" + }, + { + "name": "$return", + "type": "http", + "direction": "out" + }, + { + "name": "products", + "type": "sql", + "direction": "in", + "commandText": "SELECT * FROM [dbo].[ProductsColumnTypes]", + "commandType": "Text", + "connectionStringSetting": "SqlConnectionString" + } + ], + "disabled": false +} \ No newline at end of file diff --git a/test/Integration/test-js/GetProductsColumnTypesSerialization/index.js b/test/Integration/test-js/GetProductsColumnTypesSerialization/index.js new file mode 100644 index 000000000..732f2c8af --- /dev/null +++ b/test/Integration/test-js/GetProductsColumnTypesSerialization/index.js @@ -0,0 +1,11 @@ +/** + * This function verifies that serializing an item with various data types + * works as expected. + */ +module.exports = async function (context, req, products) { + context.log(JSON.stringify(products)); + return { + status: 200, + body: products + }; +} \ No newline at end of file diff --git a/test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj b/test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj index 54c504783..cad70ffbc 100644 --- a/test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj +++ b/test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj @@ -15,8 +15,14 @@ - - + + + + + + + Always + @@ -32,13 +38,4 @@ - - - - <_CSharpTestSqlFiles Include="Integration\test-csharp\Database\**\*.*" /> - - - - - From d36422f20d489b9ae9cf1cff7c5cce3eb86b8de7 Mon Sep 17 00:00:00 2001 From: AmeyaRele <35621237+AmeyaRele@users.noreply.github.com> Date: Tue, 20 Sep 2022 11:11:13 +0530 Subject: [PATCH 28/77] Add comments to explain LastSyncVersion selection (#358) --- src/TriggerBinding/SqlTableChangeMonitor.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 08f6f60ee..f22f97a37 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -582,8 +582,14 @@ private long RecomputeLastSyncVersion() changeVersionSet.Add(long.Parse(changeVersion, CultureInfo.InvariantCulture)); } - // If there are more than one version numbers in the set, return the second highest one. Otherwise, return + // The batch of changes are gotten in ascending order of the version number. + // With this, it is ensured that if there are multiple version numbers in the changeVersionSet, + // all the other rows with version numbers less than the highest should have either been processed or + // have leases acquired on them by another worker. + // Therefore, if there are more than one version numbers in the set, return the second highest one. Otherwise, return // the only version number in the set. + // Also this LastSyncVersion is actually updated in the GlobalState table only after verifying that the changes with + // changeVersion <= newLastSyncVersion have been processed in BuildUpdateTablesPostInvocation query. long lastSyncVersion = changeVersionSet.ElementAt(changeVersionSet.Count > 1 ? changeVersionSet.Count - 2 : 0); this._logger.LogDebugWithThreadId($"RecomputeLastSyncVersion. LastSyncVersion={lastSyncVersion} ChangeVersionSet={string.Join(",", changeVersionSet)}"); return lastSyncVersion; From 942736c422129ddcfa7d0aed2cc58deb5b86f839 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 20 Sep 2022 09:35:04 -0700 Subject: [PATCH 29/77] Fix --- src/TriggerBinding/SqlTableChangeMonitor.cs | 30 ++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 08f6f60ee..4842c7a8d 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -60,7 +60,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly IDictionary _telemetryProps; - private IReadOnlyList> _rows; + private IReadOnlyList> _rows; private int _leaseRenewalCount; private State _state; @@ -121,7 +121,7 @@ public SqlTableChangeMonitor( this._telemetryProps = telemetryProps; this._rowsLock = new SemaphoreSlim(1, 1); - this._rows = new List>(); + this._rows = new List>(); this._leaseRenewalCount = 0; this._state = State.CheckingForChanges; @@ -222,7 +222,7 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo } this._logger.LogDebugWithThreadId($"END UpdateTablesPreInvocation Duration={setLastSyncVersionDurationMs}ms"); - var rows = new List>(); + var rows = new List>(); // Use the version number to query for new changes. using (SqlCommand getChangesCommand = this.BuildGetChangesCommand(connection, transaction)) @@ -292,7 +292,7 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo { // If there's an exception in any part of the process, we want to clear all of our data in memory and // retry checking for changes again. - this._rows = new List>(); + this._rows = new List>(); this._logger.LogError($"Failed to check for changes in table '{this._userTable.FullName}' due to exception: {e.GetType()}. Exception message: {e.Message}"); TelemetryInstance.TrackException(TelemetryErrorName.GetChanges, e, this._telemetryProps); } @@ -477,7 +477,7 @@ private async Task ClearRowsAsync() this._leaseRenewalCount = 0; this._state = State.CheckingForChanges; - this._rows = new List>(); + this._rows = new List>(); this._logger.LogDebugWithThreadId("ReleaseRowsLock - ClearRows"); this._rowsLock.Release(); @@ -576,9 +576,9 @@ private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToke private long RecomputeLastSyncVersion() { var changeVersionSet = new SortedSet(); - foreach (IReadOnlyDictionary row in this._rows) + foreach (IReadOnlyDictionary row in this._rows) { - string changeVersion = row["SYS_CHANGE_VERSION"]; + string changeVersion = row["SYS_CHANGE_VERSION"].ToString(); changeVersionSet.Add(long.Parse(changeVersion, CultureInfo.InvariantCulture)); } @@ -599,13 +599,13 @@ private IReadOnlyList> ProcessChanges() { this._logger.LogDebugWithThreadId("BEGIN ProcessChanges"); var changes = new List>(); - foreach (IReadOnlyDictionary row in this._rows) + foreach (IReadOnlyDictionary row in this._rows) { SqlChangeOperation operation = GetChangeOperation(row); // If the row has been deleted, there is no longer any data for it in the user table. The best we can do // is populate the row-item with the primary key values of the row. - Dictionary item = operation == SqlChangeOperation.Delete + Dictionary item = operation == SqlChangeOperation.Delete ? this._primaryKeyColumns.ToDictionary(col => col, col => row[col]) : this._userTableColumns.ToDictionary(col => col, col => row[col]); @@ -621,9 +621,9 @@ private IReadOnlyList> ProcessChanges() /// The (combined) row from the change table and leases table /// Thrown if the value of the "SYS_CHANGE_OPERATION" column is none of "I", "U", or "D" /// SqlChangeOperation.Insert for an insert, SqlChangeOperation.Update for an update, and SqlChangeOperation.Delete for a delete - private static SqlChangeOperation GetChangeOperation(IReadOnlyDictionary row) + private static SqlChangeOperation GetChangeOperation(IReadOnlyDictionary row) { - string operation = row["SYS_CHANGE_OPERATION"]; + string operation = row["SYS_CHANGE_OPERATION"].ToString(); switch (operation) { case "I": return SqlChangeOperation.Insert; @@ -704,14 +704,14 @@ LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCo /// The transaction to add to the returned SqlCommand /// Dictionary representing the table rows on which leases should be acquired /// The SqlCommand populated with the query and appropriate parameters - private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransaction transaction, IReadOnlyList> rows) + private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransaction transaction, IReadOnlyList> rows) { var acquireLeasesQuery = new StringBuilder(); for (int rowIndex = 0; rowIndex < rows.Count; rowIndex++) { string valuesList = string.Join(", ", this._primaryKeyColumns.Select((_, colIndex) => $"@{rowIndex}_{colIndex}")); - string changeVersion = rows[rowIndex]["SYS_CHANGE_VERSION"]; + string changeVersion = rows[rowIndex]["SYS_CHANGE_VERSION"].ToString(); acquireLeasesQuery.Append($@" IF NOT EXISTS (SELECT * FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}) @@ -761,7 +761,7 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa for (int rowIndex = 0; rowIndex < this._rows.Count; rowIndex++) { - string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"]; + string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"].ToString(); releaseLeasesQuery.Append($@" SELECT @current_change_version = {SqlTriggerConstants.LeasesTableChangeVersionColumnName} @@ -843,7 +843,7 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_ /// rebuild the SqlParameters each time. /// private SqlCommand GetSqlCommandWithParameters(string commandText, SqlConnection connection, - SqlTransaction transaction, IReadOnlyList> rows) + SqlTransaction transaction, IReadOnlyList> rows) { var command = new SqlCommand(commandText, connection, transaction); From 89f79810d4e7d9e726b566b755f5f2343a286782 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 20 Sep 2022 11:04:59 -0700 Subject: [PATCH 30/77] Move test scripts --- .../Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql | 0 .../Database/Tables/ProductsWithUnsupportedColumnTypes.sql | 0 .../Database/Tables/ProductsWithoutPrimaryKey.sql | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename test/{Integration/test-csharp => }/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql (100%) rename test/{Integration/test-csharp => }/Database/Tables/ProductsWithUnsupportedColumnTypes.sql (100%) rename test/{Integration/test-csharp => }/Database/Tables/ProductsWithoutPrimaryKey.sql (100%) diff --git a/test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql b/test/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql similarity index 100% rename from test/Integration/test-csharp/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql rename to test/Database/Tables/ProductsWithReservedPrimaryKeyColumnNames.sql diff --git a/test/Integration/test-csharp/Database/Tables/ProductsWithUnsupportedColumnTypes.sql b/test/Database/Tables/ProductsWithUnsupportedColumnTypes.sql similarity index 100% rename from test/Integration/test-csharp/Database/Tables/ProductsWithUnsupportedColumnTypes.sql rename to test/Database/Tables/ProductsWithUnsupportedColumnTypes.sql diff --git a/test/Integration/test-csharp/Database/Tables/ProductsWithoutPrimaryKey.sql b/test/Database/Tables/ProductsWithoutPrimaryKey.sql similarity index 100% rename from test/Integration/test-csharp/Database/Tables/ProductsWithoutPrimaryKey.sql rename to test/Database/Tables/ProductsWithoutPrimaryKey.sql From 56d8289feb9af702c75567dc082f83ccf25d5969 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 20 Sep 2022 15:43:51 -0700 Subject: [PATCH 31/77] Constructor cleanup (#362) * Initialize vars at declaration * Remove extra ; * More cleanup * Remove configuration --- .editorconfig | 3 ++ src/TriggerBinding/SqlTableChangeMonitor.cs | 52 ++++++------------- .../SqlTriggerBindingProvider.cs | 3 +- src/TriggerBinding/SqlTriggerListener.cs | 19 +++---- 4 files changed, 27 insertions(+), 50 deletions(-) diff --git a/.editorconfig b/.editorconfig index e2f0c4ea8..ee08af36d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,6 +10,9 @@ dotnet_analyzer_diagnostic.severity = error # Namespace does not match folder structure - Ideally this should be enabled but it seems to have issues with root level files so disabling for now dotnet_diagnostic.IDE0130.severity = none +# CA1805: Do not initialize unnecessarily - It's better to be explicit when initializing vars to ensure correct value is used +dotnet_diagnostic.CA1805.severity = none + # Documentation related errors, remove once they are fixed dotnet_diagnostic.CS1591.severity = none dotnet_diagnostic.CS1573.severity = none diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index d684b5d6c..eee1a77bb 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -47,22 +47,22 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; - private readonly CancellationTokenSource _cancellationTokenSourceCheckForChanges; - private readonly CancellationTokenSource _cancellationTokenSourceRenewLeases; - private CancellationTokenSource _cancellationTokenSourceExecutor; + private readonly CancellationTokenSource _cancellationTokenSourceCheckForChanges = new CancellationTokenSource(); + private readonly CancellationTokenSource _cancellationTokenSourceRenewLeases = new CancellationTokenSource(); + private CancellationTokenSource _cancellationTokenSourceExecutor = new CancellationTokenSource(); // The semaphore gets used by lease-renewal loop to ensure that '_state' stays set to 'ProcessingChanges' while // the leases are being renewed. The change-consumption loop requires to wait for the semaphore before modifying // the value of '_state' back to 'CheckingForChanges'. Since the field '_rows' is only updated if the value of // '_state' is set to 'CheckingForChanges', this guarantees that '_rows' will stay same while it is being // iterated over inside the lease-renewal loop. - private readonly SemaphoreSlim _rowsLock; + private readonly SemaphoreSlim _rowsLock = new SemaphoreSlim(1, 1); private readonly IDictionary _telemetryProps; - private IReadOnlyList> _rows; - private int _leaseRenewalCount; - private State _state; + private IReadOnlyList> _rows = new List>(); + private int _leaseRenewalCount = 0; + private State _state = State.CheckingForChanges; /// /// Initializes a new instance of the > class. @@ -89,41 +89,23 @@ public SqlTableChangeMonitor( ILogger logger, IDictionary telemetryProps) { - _ = !string.IsNullOrEmpty(connectionString) ? true : throw new ArgumentNullException(nameof(connectionString)); - _ = !string.IsNullOrEmpty(userTable.FullName) ? true : throw new ArgumentNullException(nameof(userTable)); - _ = !string.IsNullOrEmpty(userFunctionId) ? true : throw new ArgumentNullException(nameof(userFunctionId)); - _ = !string.IsNullOrEmpty(leasesTableName) ? true : throw new ArgumentNullException(nameof(leasesTableName)); - _ = userTableColumns ?? throw new ArgumentNullException(nameof(userTableColumns)); - _ = primaryKeyColumns ?? throw new ArgumentNullException(nameof(primaryKeyColumns)); - _ = executor ?? throw new ArgumentNullException(nameof(executor)); - _ = logger ?? throw new ArgumentNullException(nameof(logger)); - - this._connectionString = connectionString; + this._connectionString = !string.IsNullOrEmpty(connectionString) ? connectionString : throw new ArgumentNullException(nameof(connectionString)); + this._userTable = !string.IsNullOrEmpty(userTable?.FullName) ? userTable : throw new ArgumentNullException(nameof(userTable)); + this._userFunctionId = !string.IsNullOrEmpty(userFunctionId) ? userFunctionId : throw new ArgumentNullException(nameof(userFunctionId)); + this._leasesTableName = !string.IsNullOrEmpty(leasesTableName) ? leasesTableName : throw new ArgumentNullException(nameof(leasesTableName)); + this._userTableColumns = userTableColumns ?? throw new ArgumentNullException(nameof(userTableColumns)); + this._primaryKeyColumns = primaryKeyColumns ?? throw new ArgumentNullException(nameof(primaryKeyColumns)); + this._executor = executor ?? throw new ArgumentNullException(nameof(executor)); + this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this._userTableId = userTableId; - this._userTable = userTable; - this._userFunctionId = userFunctionId; - this._leasesTableName = leasesTableName; - this._userTableColumns = userTableColumns; - this._primaryKeyColumns = primaryKeyColumns; // Prep search-conditions that will be used besides WHERE clause to match table rows. this._rowMatchConditions = Enumerable.Range(0, BatchSize) .Select(rowIndex => string.Join(" AND ", this._primaryKeyColumns.Select((col, colIndex) => $"{col.AsBracketQuotedString()} = @{rowIndex}_{colIndex}"))) .ToList(); - this._executor = executor; - this._logger = logger; - - this._cancellationTokenSourceCheckForChanges = new CancellationTokenSource(); - this._cancellationTokenSourceRenewLeases = new CancellationTokenSource(); - this._cancellationTokenSourceExecutor = new CancellationTokenSource(); - - this._telemetryProps = telemetryProps; - - this._rowsLock = new SemaphoreSlim(1, 1); - this._rows = new List>(); - this._leaseRenewalCount = 0; - this._state = State.CheckingForChanges; + this._telemetryProps = telemetryProps ?? new Dictionary(); #pragma warning disable CS4014 // Queue the below tasks and exit. Do not wait for their completion. _ = Task.Run(() => diff --git a/src/TriggerBinding/SqlTriggerBindingProvider.cs b/src/TriggerBinding/SqlTriggerBindingProvider.cs index 2fc6644ca..b35ed3f01 100644 --- a/src/TriggerBinding/SqlTriggerBindingProvider.cs +++ b/src/TriggerBinding/SqlTriggerBindingProvider.cs @@ -33,8 +33,7 @@ public SqlTriggerBindingProvider(IConfiguration configuration, IHostIdProvider h this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); this._hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); - _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); - this._logger = loggerFactory.CreateLogger(LogCategories.CreateTriggerCategory("Sql")); + this._logger = loggerFactory?.CreateLogger(LogCategories.CreateTriggerCategory("Sql")) ?? throw new ArgumentNullException(nameof(loggerFactory)); } /// diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 5523ffc69..377aba054 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -38,7 +38,7 @@ internal sealed class SqlTriggerListener : IListener private readonly IDictionary _telemetryProps = new Dictionary(); private SqlTableChangeMonitor _changeMonitor; - private int _listenerState; + private int _listenerState = ListenerNotStarted; /// /// Initializes a new instance of the class. @@ -50,18 +50,11 @@ internal sealed class SqlTriggerListener : IListener /// Facilitates logging of messages public SqlTriggerListener(string connectionString, string tableName, string userFunctionId, ITriggeredFunctionExecutor executor, ILogger logger) { - _ = !string.IsNullOrEmpty(connectionString) ? true : throw new ArgumentNullException(nameof(connectionString)); - _ = !string.IsNullOrEmpty(tableName) ? true : throw new ArgumentNullException(nameof(tableName)); - _ = !string.IsNullOrEmpty(userFunctionId) ? true : throw new ArgumentNullException(nameof(userFunctionId)); - _ = executor ?? throw new ArgumentNullException(nameof(executor)); - _ = logger ?? throw new ArgumentNullException(nameof(logger)); - - this._connectionString = connectionString; - this._userTable = new SqlObject(tableName); - this._userFunctionId = userFunctionId; - this._executor = executor; - this._logger = logger; - this._listenerState = ListenerNotStarted; + this._connectionString = !string.IsNullOrEmpty(connectionString) ? connectionString : throw new ArgumentNullException(nameof(connectionString)); + this._userTable = !string.IsNullOrEmpty(tableName) ? new SqlObject(tableName) : throw new ArgumentNullException(nameof(tableName)); + this._userFunctionId = !string.IsNullOrEmpty(userFunctionId) ? userFunctionId : throw new ArgumentNullException(nameof(userFunctionId)); + this._executor = executor ?? throw new ArgumentNullException(nameof(executor)); + this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public void Cancel() From 2717881bf6d9eb87a11149f2ed8d22a340e56872 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Thu, 22 Sep 2022 10:52:40 -0700 Subject: [PATCH 32/77] Add configuration for batch size/polling interval (#364) * Add configuration for batch size/polling interval * PR comments --- README.md | 18 ++++++ src/TriggerBinding/SqlTableChangeMonitor.cs | 55 +++++++++++++------ src/TriggerBinding/SqlTriggerBinding.cs | 8 ++- .../SqlTriggerBindingProvider.cs | 4 +- src/TriggerBinding/SqlTriggerConstants.cs | 3 + src/TriggerBinding/SqlTriggerListener.cs | 9 ++- src/Utils.cs | 5 ++ test/.editorconfig | 3 +- test/Integration/IntegrationTestBase.cs | 7 ++- .../SqlTriggerBindingIntegrationTests.cs | 43 +++++++++++++++ .../ProductsTriggerWithValidation.cs | 33 +++++++++++ 11 files changed, 163 insertions(+), 25 deletions(-) create mode 100644 test/Integration/test-csharp/ProductsTriggerWithValidation.cs diff --git a/README.md b/README.md index 9fd3cb0d2..fb6166bac 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,10 @@ Azure SQL bindings for Azure Functions are supported for: - [Python functions](#python-functions) - [Input Binding Tutorial](#input-binding-tutorial-2) - [Output Binding Tutorial](#output-binding-tutorial-2) + - [Configuration](#configuration) + - [Trigger Binding Configuration](#trigger-binding-configuration) + - [Sql_Trigger_BatchSize](#sql_trigger_batchsize) + - [Sql_Trigger_PollingIntervalMs](#sql_trigger_pollingintervalms) - [More Samples](#more-samples) - [Input Binding](#input-binding) - [Query String](#query-string) @@ -554,6 +558,20 @@ Note: This tutorial requires that a SQL database is setup as shown in [Create a - Hit 'F5' to run your code. Click the link to upsert the output array values in your SQL table. Your upserted values should launch in the browser. - Congratulations! You have successfully created your first SQL output binding! Checkout [Output Binding](#Output-Binding) for more information on how to use it and explore on your own! +## Configuration + +This section goes over some of the configuration values you can use to customize the SQL bindings. See [How to Use Azure Function App Settings](https://learn.microsoft.com/azure/azure-functions/functions-how-to-use-azure-function-app-settings) to learn more. + +### Trigger Binding Configuration + +#### Sql_Trigger_BatchSize + +This controls the number of changes processed at once before being sent to the triggered function. + +#### Sql_Trigger_PollingIntervalMs + +This controls the delay in milliseconds between processing each batch of changes. + ## More Samples ### Input Binding diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index eee1a77bb..3d161a708 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -16,6 +16,7 @@ using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using Newtonsoft.Json; +using Microsoft.Extensions.Configuration; namespace Microsoft.Azure.WebJobs.Extensions.Sql { @@ -25,17 +26,25 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql /// POCO class representing the row in the user table internal sealed class SqlTableChangeMonitor : IDisposable { - public const int BatchSize = 10; - public const int PollingIntervalInSeconds = 5; - public const int MaxAttemptCount = 5; - - // Leases are held for approximately (LeaseRenewalIntervalInSeconds * MaxLeaseRenewalCount) seconds. It is + #region Constants + /// + /// The maximum number of times we'll attempt to process a change before giving up + /// + private const int MaxChangeProcessAttemptCount = 5; + /// + /// The maximum number of times that we'll attempt to renew a lease be + /// + /// + /// Leases are held for approximately (LeaseRenewalIntervalInSeconds * MaxLeaseRenewalCount) seconds. It is // required to have at least one of (LeaseIntervalInSeconds / LeaseRenewalIntervalInSeconds) attempts to // renew the lease succeed to prevent it from expiring. - public const int MaxLeaseRenewalCount = 10; - public const int LeaseIntervalInSeconds = 60; - public const int LeaseRenewalIntervalInSeconds = 15; - public const int MaxRetryReleaseLeases = 3; + // + private const int MaxLeaseRenewalCount = 10; + private const int LeaseIntervalInSeconds = 60; + private const int LeaseRenewalIntervalInSeconds = 15; + private const int MaxRetryReleaseLeases = 3; + #endregion Constants + private readonly string _connectionString; private readonly int _userTableId; private readonly SqlObject _userTable; @@ -46,6 +55,14 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly IReadOnlyList _rowMatchConditions; private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; + /// + /// Number of changes to process in each iteration of the loop + /// + private readonly int _batchSize = 10; + /// + /// Delay in ms between processing each batch of changes + /// + private readonly int _pollingIntervalInMs = 5000; private readonly CancellationTokenSource _cancellationTokenSourceCheckForChanges = new CancellationTokenSource(); private readonly CancellationTokenSource _cancellationTokenSourceRenewLeases = new CancellationTokenSource(); @@ -87,6 +104,7 @@ public SqlTableChangeMonitor( IReadOnlyList primaryKeyColumns, ITriggeredFunctionExecutor executor, ILogger logger, + IConfiguration configuration, IDictionary telemetryProps) { this._connectionString = !string.IsNullOrEmpty(connectionString) ? connectionString : throw new ArgumentNullException(nameof(connectionString)); @@ -99,9 +117,11 @@ public SqlTableChangeMonitor( this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); this._userTableId = userTableId; - + // Check if there's config settings to override the default batch size/polling interval values + this._batchSize = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize) ?? this._batchSize; + this._pollingIntervalInMs = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_PollingInterval) ?? this._pollingIntervalInMs; // Prep search-conditions that will be used besides WHERE clause to match table rows. - this._rowMatchConditions = Enumerable.Range(0, BatchSize) + this._rowMatchConditions = Enumerable.Range(0, this._batchSize) .Select(rowIndex => string.Join(" AND ", this._primaryKeyColumns.Select((col, colIndex) => $"{col.AsBracketQuotedString()} = @{rowIndex}_{colIndex}"))) .ToList(); @@ -122,7 +142,7 @@ public void Dispose() } /// - /// Executed once every period. If the state of the change monitor is + /// Executed once every period. If the state of the change monitor is /// , then the method query the change/leases tables for changes on the /// user's table. If any are found, the state of the change monitor is transitioned to /// and the user's function is executed with the found changes. If the @@ -131,7 +151,7 @@ public void Dispose() /// private async Task RunChangeConsumptionLoopAsync() { - this._logger.LogDebugWithThreadId("Starting change consumption loop."); + this._logger.LogInformationWithThreadId($"Starting change consumption loop. BatchSize: {this._batchSize} PollingIntervalMs: {this._pollingIntervalInMs}"); try { @@ -153,7 +173,8 @@ private async Task RunChangeConsumptionLoopAsync() await this.ProcessTableChangesAsync(connection, token); } this._logger.LogDebugWithThreadId("END CheckingForChanges"); - await Task.Delay(TimeSpan.FromSeconds(PollingIntervalInSeconds), token); + this._logger.LogDebugWithThreadId($"Delaying for {this._pollingIntervalInMs}ms"); + await Task.Delay(TimeSpan.FromMilliseconds(this._pollingIntervalInMs), token); } } } @@ -666,7 +687,7 @@ private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransacti FROM {SqlTriggerConstants.GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; - SELECT TOP {BatchSize} + SELECT TOP {this._batchSize} {selectList}, c.SYS_CHANGE_VERSION, c.SYS_CHANGE_OPERATION, l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName}, l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName}, l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} @@ -677,7 +698,7 @@ LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCo (l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NULL AND (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.SYS_CHANGE_VERSION) OR l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND - (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxAttemptCount}) + (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}) ORDER BY c.SYS_CHANGE_VERSION ASC; "; @@ -798,7 +819,7 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_ ((l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} != c.SYS_CHANGE_VERSION OR l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NOT NULL) AND - (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxAttemptCount}))) AS Changes + (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}))) AS Changes IF @unprocessed_changes = 0 AND @current_last_sync_version < {newLastSyncVersion} BEGIN diff --git a/src/TriggerBinding/SqlTriggerBinding.cs b/src/TriggerBinding/SqlTriggerBinding.cs index b04fd95c4..841c8a79e 100644 --- a/src/TriggerBinding/SqlTriggerBinding.cs +++ b/src/TriggerBinding/SqlTriggerBinding.cs @@ -15,6 +15,7 @@ using Microsoft.Azure.WebJobs.Host.Protocols; using Microsoft.Azure.WebJobs.Host.Triggers; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Configuration; namespace Microsoft.Azure.WebJobs.Extensions.Sql { @@ -29,6 +30,7 @@ internal sealed class SqlTriggerBinding : ITriggerBinding private readonly ParameterInfo _parameter; private readonly IHostIdProvider _hostIdProvider; private readonly ILogger _logger; + private readonly IConfiguration _configuration; private static readonly IReadOnlyDictionary _emptyBindingContract = new Dictionary(); private static readonly IReadOnlyDictionary _emptyBindingData = new Dictionary(); @@ -41,13 +43,15 @@ internal sealed class SqlTriggerBinding : ITriggerBinding /// Trigger binding parameter information /// Provider of unique host identifier /// Facilitates logging of messages - public SqlTriggerBinding(string connectionString, string tableName, ParameterInfo parameter, IHostIdProvider hostIdProvider, ILogger logger) + /// Provides configuration values + public SqlTriggerBinding(string connectionString, string tableName, ParameterInfo parameter, IHostIdProvider hostIdProvider, ILogger logger, IConfiguration configuration) { this._connectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString)); this._tableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); this._parameter = parameter ?? throw new ArgumentNullException(nameof(parameter)); this._hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); } /// @@ -68,7 +72,7 @@ public async Task CreateListenerAsync(ListenerFactoryContext context) _ = context ?? throw new ArgumentNullException(nameof(context), "Missing listener context"); string userFunctionId = await this.GetUserFunctionIdAsync(); - return new SqlTriggerListener(this._connectionString, this._tableName, userFunctionId, context.Executor, this._logger); + return new SqlTriggerListener(this._connectionString, this._tableName, userFunctionId, context.Executor, this._logger, this._configuration); } public ParameterDescriptor ToParameterDescriptor() diff --git a/src/TriggerBinding/SqlTriggerBindingProvider.cs b/src/TriggerBinding/SqlTriggerBindingProvider.cs index b35ed3f01..17afa4cc3 100644 --- a/src/TriggerBinding/SqlTriggerBindingProvider.cs +++ b/src/TriggerBinding/SqlTriggerBindingProvider.cs @@ -78,10 +78,10 @@ public Task TryCreateAsync(TriggerBindingProviderContext contex Type userType = parameter.ParameterType.GetGenericArguments()[0].GetGenericArguments()[0]; Type bindingType = typeof(SqlTriggerBinding<>).MakeGenericType(userType); - var constructorParameterTypes = new Type[] { typeof(string), typeof(string), typeof(ParameterInfo), typeof(IHostIdProvider), typeof(ILogger) }; + var constructorParameterTypes = new Type[] { typeof(string), typeof(string), typeof(ParameterInfo), typeof(IHostIdProvider), typeof(ILogger), typeof(IConfiguration) }; ConstructorInfo bindingConstructor = bindingType.GetConstructor(constructorParameterTypes); - object[] constructorParameterValues = new object[] { connectionString, attribute.TableName, parameter, this._hostIdProvider, this._logger }; + object[] constructorParameterValues = new object[] { connectionString, attribute.TableName, parameter, this._hostIdProvider, this._logger, this._configuration }; var triggerBinding = (ITriggerBinding)bindingConstructor.Invoke(constructorParameterValues); return Task.FromResult(triggerBinding); diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index 2f37d866f..8c627202d 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -14,5 +14,8 @@ internal static class SqlTriggerConstants public const string LeasesTableChangeVersionColumnName = "_az_func_ChangeVersion"; public const string LeasesTableAttemptCountColumnName = "_az_func_AttemptCount"; public const string LeasesTableLeaseExpirationTimeColumnName = "_az_func_LeaseExpirationTime"; + + public const string ConfigKey_SqlTrigger_BatchSize = "Sql_Trigger_BatchSize"; + public const string ConfigKey_SqlTrigger_PollingInterval = "Sql_Trigger_PollingIntervalMs"; } } \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 377aba054..d1d95bb7d 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -14,6 +14,7 @@ using Microsoft.Azure.WebJobs.Host.Listeners; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Configuration; namespace Microsoft.Azure.WebJobs.Extensions.Sql { @@ -34,6 +35,7 @@ internal sealed class SqlTriggerListener : IListener private readonly string _userFunctionId; private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; + private readonly IConfiguration _configuration; private readonly IDictionary _telemetryProps = new Dictionary(); @@ -48,13 +50,15 @@ internal sealed class SqlTriggerListener : IListener /// Unique identifier for the user function /// Defines contract for triggering user function /// Facilitates logging of messages - public SqlTriggerListener(string connectionString, string tableName, string userFunctionId, ITriggeredFunctionExecutor executor, ILogger logger) + /// Provides configuration values + public SqlTriggerListener(string connectionString, string tableName, string userFunctionId, ITriggeredFunctionExecutor executor, ILogger logger, IConfiguration configuration) { this._connectionString = !string.IsNullOrEmpty(connectionString) ? connectionString : throw new ArgumentNullException(nameof(connectionString)); this._userTable = !string.IsNullOrEmpty(tableName) ? new SqlObject(tableName) : throw new ArgumentNullException(nameof(tableName)); this._userFunctionId = !string.IsNullOrEmpty(userFunctionId) ? userFunctionId : throw new ArgumentNullException(nameof(userFunctionId)); this._executor = executor ?? throw new ArgumentNullException(nameof(executor)); this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); } public void Cancel() @@ -122,6 +126,7 @@ public async Task StartAsync(CancellationToken cancellationToken) primaryKeyColumns.Select(col => col.name).ToList(), this._executor, this._logger, + this._configuration, this._telemetryProps); this._listenerState = ListenerStarted; @@ -467,7 +472,7 @@ PRIMARY KEY ({primaryKeys}) } /// - /// Clears the current telemetry property dictionary and initializes the default initial properties. + /// Clears the current telemetry property dictionary and initializes the default initial properties. /// private void InitializeTelemetryProps() { diff --git a/src/Utils.cs b/src/Utils.cs index d05d01a9a..0bff69855 100644 --- a/src/Utils.cs +++ b/src/Utils.cs @@ -99,5 +99,10 @@ public static void LogDebugWithThreadId(this ILogger logger, string message, par { logger.LogDebug($"TID:{Environment.CurrentManagedThreadId} {message}", args); } + + public static void LogInformationWithThreadId(this ILogger logger, string message, params object[] args) + { + logger.LogInformation($"TID:{Environment.CurrentManagedThreadId} {message}", args); + } } } diff --git a/test/.editorconfig b/test/.editorconfig index 4386dd31a..0082aa2a0 100644 --- a/test/.editorconfig +++ b/test/.editorconfig @@ -5,4 +5,5 @@ # Disabled dotnet_diagnostic.CA1309.severity = silent # Use ordinal StringComparison - this isn't important for tests and just adds clutter dotnet_diagnostic.CA1305.severity = silent # Specify IFormatProvider - this isn't important for tests and just adds clutter -dotnet_diagnostic.CA1707.severity = silent # Identifiers should not contain underscores - this helps make test names more readable \ No newline at end of file +dotnet_diagnostic.CA1707.severity = silent # Identifiers should not contain underscores - this helps make test names more readable +dotnet_diagnostic.CA2201.severity = silent # Do not raise reserved exception types - tests can throw whatever they want \ No newline at end of file diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index 665fbf3fa..63cdac911 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -17,6 +17,7 @@ using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; using Microsoft.AspNetCore.WebUtilities; using System.Collections.Generic; +using System.Linq; namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration { @@ -175,7 +176,7 @@ protected void StartAzurite() /// - The functionName is different than its route.
/// - You can start multiple functions by passing in a space-separated list of function names.
/// - protected void StartFunctionHost(string functionName, SupportedLanguages language, bool useTestFolder = false, DataReceivedEventHandler customOutputHandler = null) + protected void StartFunctionHost(string functionName, SupportedLanguages language, bool useTestFolder = false, DataReceivedEventHandler customOutputHandler = null, IDictionary environmentVariables = null) { string workingDirectory = useTestFolder ? GetPathToBin() : Path.Combine(GetPathToBin(), "SqlExtensionSamples", Enum.GetName(typeof(SupportedLanguages), language)); if (!Directory.Exists(workingDirectory)) @@ -194,6 +195,10 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag RedirectStandardError = true, UseShellExecute = false }; + if (environmentVariables != null) + { + environmentVariables.ToList().ForEach(ev => startInfo.EnvironmentVariables[ev.Key] = ev.Value); + } this.LogOutput($"Starting {startInfo.FileName} {startInfo.Arguments} in {startInfo.WorkingDirectory}"); this.FunctionHost = new Process { diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index ccea60489..7eca03418 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -55,6 +55,49 @@ public async Task SingleOperationTriggerTest() changes.Clear(); } + /// + /// Verifies that manually setting the batch size correctly changes the number of changes processed at once + /// + [Fact] + public async Task BatchSizeOverrideTriggerTest() + { + this.EnableChangeTrackingForTable("Products"); + this.StartFunctionHost(nameof(ProductsTriggerWithValidation), Common.SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { + { "TEST_EXPECTED_BATCH_SIZE", "20" }, + { "Sql_Trigger_BatchSize", "20" } + }); + + var changes = new List>(); + this.MonitorProductChanges(changes, "SQL Changes: "); + + // Considering the polling interval of 5 seconds and batch-size of 20, it should take around 10 seconds to + // process 40 insert operations. + this.InsertProducts(1, 40); + await Task.Delay(TimeSpan.FromSeconds(12)); + ValidateProductChanges(changes, 1, 40, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); + } + + /// + /// Verifies that manually setting the polling interval correctly changes the delay between processing each batch of changes + /// + [Fact] + public async Task PollingIntervalOverrideTriggerTest() + { + this.EnableChangeTrackingForTable("Products"); + this.StartFunctionHost(nameof(ProductsTriggerWithValidation), Common.SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { + { "Sql_Trigger_PollingIntervalMs", "100" } + }); + + var changes = new List>(); + this.MonitorProductChanges(changes, "SQL Changes: "); + + // Considering the polling interval of 100ms and batch-size of 10, it should take around .5 second to + // process 50 insert operations. + this.InsertProducts(1, 50); + await Task.Delay(TimeSpan.FromSeconds(1)); + ValidateProductChanges(changes, 1, 50, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); + } + /// /// Verifies that if several changes have happened to the table row since last invocation, then a single net diff --git a/test/Integration/test-csharp/ProductsTriggerWithValidation.cs b/test/Integration/test-csharp/ProductsTriggerWithValidation.cs new file mode 100644 index 000000000..7fca5cbf9 --- /dev/null +++ b/test/Integration/test-csharp/ProductsTriggerWithValidation.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration +{ + public static class ProductsTriggerWithValidation + { + /// + /// Simple trigger function with additional logic to allow for verifying that the expected number + /// of changes was recieved in each batch. + /// + [FunctionName(nameof(ProductsTriggerWithValidation))] + public static void Run( + [SqlTrigger("[dbo].[Products]", ConnectionStringSetting = "SqlConnectionString")] + IReadOnlyList> changes, + ILogger logger) + { + string expectedBatchSize = Environment.GetEnvironmentVariable("TEST_EXPECTED_BATCH_SIZE"); + if (!string.IsNullOrEmpty(expectedBatchSize) && int.Parse(expectedBatchSize) != changes.Count) + { + throw new Exception($"Invalid batch size, got {changes.Count} changes but expected {expectedBatchSize}"); + } + // The output is used to inspect the trigger binding parameter in test methods. + logger.LogInformation("SQL Changes: " + JsonConvert.SerializeObject(changes)); + } + } +} From e82dfe3236d8c0c93d7a76bd7829e4df8282d864 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Mon, 26 Sep 2022 10:58:40 -0700 Subject: [PATCH 33/77] Refactor how trigger integration tests wait for changes (#368) --- test/Common/TestUtils.cs | 18 + .../SqlTriggerBindingIntegrationTests.cs | 366 +++++++++++++----- 2 files changed, 277 insertions(+), 107 deletions(-) diff --git a/test/Common/TestUtils.cs b/test/Common/TestUtils.cs index 38330e33d..ab45410a3 100644 --- a/test/Common/TestUtils.cs +++ b/test/Common/TestUtils.cs @@ -6,6 +6,7 @@ using System.Data; using System.Diagnostics; using System.Threading; +using System.Threading.Tasks; namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common { @@ -159,5 +160,22 @@ public static string CleanJsonString(string jsonStr) { return jsonStr.Trim().Replace(" ", "").Replace(Environment.NewLine, ""); } + + public static async Task TimeoutAfter(this Task task, TimeSpan timeout, string message = "The operation has timed out.") + { + + using var timeoutCancellationTokenSource = new CancellationTokenSource(); + + Task completedTask = await Task.WhenAny(task, Task.Delay(timeout, timeoutCancellationTokenSource.Token)); + if (completedTask == task) + { + timeoutCancellationTokenSource.Cancel(); + return await task; // Very important in order to propagate exceptions + } + else + { + throw new TimeoutException(message); + } + } } } diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 7eca03418..01440c41a 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; @@ -29,30 +30,40 @@ public SqlTriggerBindingIntegrationTests(ITestOutputHelper output) : base(output public async Task SingleOperationTriggerTest() { this.EnableChangeTrackingForTable("Products"); - this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); - - var changes = new List>(); - this.MonitorProductChanges(changes, "SQL Changes: "); + this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); // Considering the polling interval of 5 seconds and batch-size of 10, it should take around 15 seconds to - // process 30 insert operations. Similar reasoning is used to set delays for update and delete operations. - this.InsertProducts(1, 30); - await Task.Delay(TimeSpan.FromSeconds(20)); - ValidateProductChanges(changes, 1, 30, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); - changes.Clear(); + // process 30 insert operations. An extra 5sec is added as a buffer to the timeout. + // Similar reasoning is used to set delays for update and delete operations. + await this.WaitForProductChanges( + 1, + 30, + SqlChangeOperation.Insert, + () => { this.InsertProducts(1, 30); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + 20000); // All table columns (not just the columns that were updated) would be returned for update operation. - this.UpdateProducts(1, 20); - await Task.Delay(TimeSpan.FromSeconds(15)); - ValidateProductChanges(changes, 1, 20, SqlChangeOperation.Update, id => $"Updated Product {id}", id => id * 100); - changes.Clear(); + await this.WaitForProductChanges( + 1, + 20, + SqlChangeOperation.Update, + () => { this.UpdateProducts(1, 20); return Task.CompletedTask; }, + id => $"Updated Product {id}", + id => id * 100, + 20000); // The properties corresponding to non-primary key columns would be set to the C# type's default values // (null and 0) for delete operation. - this.DeleteProducts(11, 30); - await Task.Delay(TimeSpan.FromSeconds(15)); - ValidateProductChanges(changes, 11, 30, SqlChangeOperation.Delete, _ => null, _ => 0); - changes.Clear(); + await this.WaitForProductChanges( + 11, + 30, + SqlChangeOperation.Delete, + () => { this.DeleteProducts(11, 30); return Task.CompletedTask; }, + _ => null, + _ => 0, + 20000); } /// @@ -62,19 +73,21 @@ public async Task SingleOperationTriggerTest() public async Task BatchSizeOverrideTriggerTest() { this.EnableChangeTrackingForTable("Products"); - this.StartFunctionHost(nameof(ProductsTriggerWithValidation), Common.SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { + this.StartFunctionHost(nameof(ProductsTriggerWithValidation), SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { { "TEST_EXPECTED_BATCH_SIZE", "20" }, { "Sql_Trigger_BatchSize", "20" } }); - var changes = new List>(); - this.MonitorProductChanges(changes, "SQL Changes: "); - - // Considering the polling interval of 5 seconds and batch-size of 20, it should take around 10 seconds to - // process 40 insert operations. - this.InsertProducts(1, 40); - await Task.Delay(TimeSpan.FromSeconds(12)); - ValidateProductChanges(changes, 1, 40, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); + // Considering the polling interval of 5 seconds and batch-size of 20, it should take around 10sec to + // process 40 insert operations. Added buffer time to timeout for total of 15sec. + await this.WaitForProductChanges( + 1, + 40, + SqlChangeOperation.Insert, + () => { this.InsertProducts(1, 40); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + 15000); } /// @@ -84,18 +97,20 @@ public async Task BatchSizeOverrideTriggerTest() public async Task PollingIntervalOverrideTriggerTest() { this.EnableChangeTrackingForTable("Products"); - this.StartFunctionHost(nameof(ProductsTriggerWithValidation), Common.SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { + this.StartFunctionHost(nameof(ProductsTriggerWithValidation), SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { { "Sql_Trigger_PollingIntervalMs", "100" } }); - var changes = new List>(); - this.MonitorProductChanges(changes, "SQL Changes: "); - // Considering the polling interval of 100ms and batch-size of 10, it should take around .5 second to - // process 50 insert operations. - this.InsertProducts(1, 50); - await Task.Delay(TimeSpan.FromSeconds(1)); - ValidateProductChanges(changes, 1, 50, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); + // process 50 insert operations. Added buffer time to timeout for total of 2sec. + await this.WaitForProductChanges( + 1, + 50, + SqlChangeOperation.Insert, + () => { this.InsertProducts(1, 50); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + 2000); } @@ -107,36 +122,69 @@ public async Task PollingIntervalOverrideTriggerTest() public async Task MultiOperationTriggerTest() { this.EnableChangeTrackingForTable("Products"); - this.StartFunctionHost(nameof(ProductsTrigger), Common.SupportedLanguages.CSharp); - - var changes = new List>(); - this.MonitorProductChanges(changes, "SQL Changes: "); - - // Insert + multiple updates to a row are treated as single insert with latest row values. - this.InsertProducts(1, 5); - this.UpdateProducts(1, 5); - this.UpdateProducts(1, 5); - await Task.Delay(TimeSpan.FromSeconds(6)); - ValidateProductChanges(changes, 1, 5, SqlChangeOperation.Insert, id => $"Updated Updated Product {id}", id => id * 100); - changes.Clear(); - - // Multiple updates to a row are treated as single update with latest row values. - this.InsertProducts(6, 10); - await Task.Delay(TimeSpan.FromSeconds(6)); - changes.Clear(); - this.UpdateProducts(6, 10); - this.UpdateProducts(6, 10); - await Task.Delay(TimeSpan.FromSeconds(6)); - ValidateProductChanges(changes, 6, 10, SqlChangeOperation.Update, id => $"Updated Updated Product {id}", id => id * 100); - changes.Clear(); - - // Insert + (zero or more updates) + delete to a row are treated as single delete with default values for non-primary columns. - this.InsertProducts(11, 20); - this.UpdateProducts(11, 20); - this.DeleteProducts(11, 20); - await Task.Delay(TimeSpan.FromSeconds(6)); - ValidateProductChanges(changes, 11, 20, SqlChangeOperation.Delete, _ => null, _ => 0); - changes.Clear(); + this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); + + // 1. Insert + multiple updates to a row are treated as single insert with latest row values. + await this.WaitForProductChanges( + 1, + 5, + SqlChangeOperation.Insert, + () => + { + this.InsertProducts(1, 5); + this.UpdateProducts(1, 5); + this.UpdateProducts(1, 5); + return Task.CompletedTask; + }, + id => $"Updated Updated Product {id}", + id => id * 100, + 10000); + + // 2. Multiple updates to a row are treated as single update with latest row values. + // First insert items and wait for those changes to be sent + await this.WaitForProductChanges( + 6, + 10, + SqlChangeOperation.Insert, + () => + { + this.InsertProducts(6, 10); + return Task.CompletedTask; + }, + id => $"Product {id}", + id => id * 100, + 10000); + + // Now do multiple updates at once and verify the updates are batched together + await this.WaitForProductChanges( + 6, + 10, + SqlChangeOperation.Update, + () => + { + this.UpdateProducts(6, 10); + this.UpdateProducts(6, 10); + return Task.CompletedTask; + }, + id => $"Updated Updated Product {id}", + id => id * 100, + 10000); + + // 3. Insert + (zero or more updates) + delete to a row are treated as single delete with default values for non-primary columns. + await this.WaitForProductChanges( + 11, + 20, + SqlChangeOperation.Delete, + () => + { + this.InsertProducts(11, 20); + this.UpdateProducts(11, 20); + this.DeleteProducts(11, 20); + return Task.CompletedTask; + }, + _ => null, + _ => 0, + 10000); } @@ -146,43 +194,121 @@ public async Task MultiOperationTriggerTest() [Fact] public async Task MultiFunctionTriggerTest() { + const string Trigger1Changes = "Trigger1 Changes: "; + const string Trigger2Changes = "Trigger2 Changes: "; + this.EnableChangeTrackingForTable("Products"); string functionList = $"{nameof(MultiFunctionTrigger.MultiFunctionTrigger1)} {nameof(MultiFunctionTrigger.MultiFunctionTrigger2)}"; - this.StartFunctionHost(functionList, Common.SupportedLanguages.CSharp, useTestFolder: true); - - var changes1 = new List>(); - var changes2 = new List>(); - - this.MonitorProductChanges(changes1, "Trigger1 Changes: "); - this.MonitorProductChanges(changes2, "Trigger2 Changes: "); + this.StartFunctionHost(functionList, SupportedLanguages.CSharp, useTestFolder: true); // Considering the polling interval of 5 seconds and batch-size of 10, it should take around 15 seconds to - // process 30 insert operations for each trigger-listener. Similar reasoning is used to set delays for - // update and delete operations. + // process 30 insert operations for each trigger-listener. Buffer of 5sec added to timeout. + // Similar reasoning is used to set delays for update and delete operations. + + // 1. INSERT + // Set up monitoring for Trigger 1... + Task changes1Task = this.WaitForProductChanges( + 1, + 30, + SqlChangeOperation.Insert, + () => + { + return Task.CompletedTask; + }, + id => $"Product {id}", + id => id * 100, + 20000, + Trigger1Changes + ); + + // Set up monitoring for Trigger 2... + Task changes2Task = this.WaitForProductChanges( + 1, + 30, + SqlChangeOperation.Insert, + () => + { + return Task.CompletedTask; + }, + id => $"Product {id}", + id => id * 100, + 25000, + Trigger2Changes + ); + + // Now that monitoring is set up make the changes and then wait for the monitoring tasks to see them and complete this.InsertProducts(1, 30); - await Task.Delay(TimeSpan.FromSeconds(20)); - ValidateProductChanges(changes1, 1, 30, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); - ValidateProductChanges(changes2, 1, 30, SqlChangeOperation.Insert, id => $"Product {id}", id => id * 100); - changes1.Clear(); - changes2.Clear(); + await Task.WhenAll(changes1Task, changes2Task); + // 2. UPDATE // All table columns (not just the columns that were updated) would be returned for update operation. + // Set up monitoring for Trigger 1... + changes1Task = this.WaitForProductChanges( + 1, + 20, + SqlChangeOperation.Update, + () => + { + return Task.CompletedTask; + }, + id => $"Updated Product {id}", + id => id * 100, + 25000, + Trigger1Changes); + + // Set up monitoring for Trigger 2... + changes2Task = this.WaitForProductChanges( + 1, + 20, + SqlChangeOperation.Update, + () => + { + return Task.CompletedTask; + }, + id => $"Updated Product {id}", + id => id * 100, + 25000, + Trigger2Changes); + + // Now that monitoring is set up make the changes and then wait for the monitoring tasks to see them and complete this.UpdateProducts(1, 20); - await Task.Delay(TimeSpan.FromSeconds(15)); - ValidateProductChanges(changes1, 1, 20, SqlChangeOperation.Update, id => $"Updated Product {id}", id => id * 100); - ValidateProductChanges(changes2, 1, 20, SqlChangeOperation.Update, id => $"Updated Product {id}", id => id * 100); - changes1.Clear(); - changes2.Clear(); + await Task.WhenAll(changes1Task, changes2Task); + // 3. DELETE // The properties corresponding to non-primary key columns would be set to the C# type's default values // (null and 0) for delete operation. + // Set up monitoring for Trigger 1... + changes1Task = this.WaitForProductChanges( + 11, + 30, + SqlChangeOperation.Delete, + () => + { + return Task.CompletedTask; + }, + _ => null, + _ => 0, + 25000, + Trigger1Changes); + + // Set up monitoring for Trigger 2... + changes2Task = this.WaitForProductChanges( + 11, + 30, + SqlChangeOperation.Delete, + () => + { + return Task.CompletedTask; + }, + _ => null, + _ => 0, + 25000, + Trigger2Changes); + + // Now that monitoring is set up make the changes and then wait for the monitoring tasks to see them and complete this.DeleteProducts(11, 30); - await Task.Delay(TimeSpan.FromSeconds(15)); - ValidateProductChanges(changes1, 11, 30, SqlChangeOperation.Delete, _ => null, _ => 0); - ValidateProductChanges(changes2, 11, 30, SqlChangeOperation.Delete, _ => null, _ => 0); - changes1.Clear(); - changes2.Clear(); + await Task.WhenAll(changes1Task, changes2Task); } /// @@ -304,28 +430,54 @@ private void DeleteProducts(int first_id, int last_id) "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(first_id, count)) + ");"); } - private static void ValidateProductChanges(List> changes, int first_id, int last_id, - SqlChangeOperation operation, Func getName, Func getCost) + private async Task WaitForProductChanges( + int firstId, + int lastId, + SqlChangeOperation operation, + Func actions, + Func getName, + Func getCost, + int timeoutMs, + string messagePrefix = "SQL Changes: ") { - int count = last_id - first_id + 1; - Assert.Equal(count, changes.Count); + var expectedIds = Enumerable.Range(firstId, lastId - firstId + 1).ToHashSet(); + int index = 0; - // Since the table rows are changed with a single SQL statement, the changes are not guaranteed to arrive in - // ProductID-order. Occasionally, we find the items in the second batch are passed to the user function in - // reverse order, which is an expected behavior. - IEnumerable> orderedChanges = changes.OrderBy(change => change.Item.ProductID); + var taskCompletion = new TaskCompletionSource(); - int id = first_id; - foreach (SqlChange change in orderedChanges) + void MonitorOutputData(object sender, DataReceivedEventArgs e) { - Assert.Equal(operation, change.Operation); - Product product = change.Item; - Assert.NotNull(product); - Assert.Equal(id, product.ProductID); - Assert.Equal(getName(id), product.Name); - Assert.Equal(getCost(id), product.Cost); - id += 1; - } + if (e.Data != null && (index = e.Data.IndexOf(messagePrefix, StringComparison.Ordinal)) >= 0) + { + string json = e.Data[(index + messagePrefix.Length)..]; + IReadOnlyList> changes = JsonConvert.DeserializeObject>>(json); + foreach (SqlChange change in changes) + { + Assert.Equal(operation, change.Operation); // Expected change operation + Product product = change.Item; + Assert.NotNull(product); // Product deserialized correctly + Assert.Contains(product.ProductID, expectedIds); // We haven't seen this product ID yet, and it's one we expected to see + expectedIds.Remove(product.ProductID); + Assert.Equal(getName(product.ProductID), product.Name); // The product has the expected name + Assert.Equal(getCost(product.ProductID), product.Cost); // The product has the expected cost + } + if (expectedIds.Count == 0) + { + taskCompletion.SetResult(true); + } + } + }; + // Set up listener for the changes coming in + this.FunctionHost.OutputDataReceived += MonitorOutputData; + + // Now that we've set up our listener trigger the actions to monitor + await actions(); + + // Now wait until either we timeout or we've gotten all the expected changes, whichever comes first + await taskCompletion.Task.TimeoutAfter(TimeSpan.FromMilliseconds(timeoutMs), $"Timed out waiting for {operation} changes."); + + // Unhook handler since we're done monitoring these changes so we aren't checking other changes done later + this.FunctionHost.OutputDataReceived -= MonitorOutputData; } /// @@ -355,7 +507,7 @@ void OutputHandler(object sender, DataReceivedEventArgs e) }; // All trigger integration tests are only using C# functions for testing at the moment. - this.StartFunctionHost(functionName, Common.SupportedLanguages.CSharp, useTestFolder, OutputHandler); + this.StartFunctionHost(functionName, SupportedLanguages.CSharp, useTestFolder, OutputHandler); // The functions host generally logs the error message within a second after starting up. const int BufferTimeForErrorInSeconds = 15; From f840a272a3a66977c03673176165b147ef165e7e Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 27 Sep 2022 08:04:14 -0700 Subject: [PATCH 34/77] Always disable telemetry during integration tests (#372) --- test/Integration/IntegrationTestBase.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index 63cdac911..95170d4cd 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -18,6 +18,7 @@ using Microsoft.AspNetCore.WebUtilities; using System.Collections.Generic; using System.Linq; +using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry; namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration { @@ -199,6 +200,10 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag { environmentVariables.ToList().ForEach(ev => startInfo.EnvironmentVariables[ev.Key] = ev.Value); } + + // Always disable telemetry during test runs + startInfo.EnvironmentVariables[TelemetryOptoutEnvVar] = "1"; + this.LogOutput($"Starting {startInfo.FileName} {startInfo.Arguments} in {startInfo.WorkingDirectory}"); this.FunctionHost = new Process { From 5d8a8b6ec2f65473ccdeca5fd67954b778c3fe9c Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 27 Sep 2022 08:23:42 -0700 Subject: [PATCH 35/77] Check for reserved column names on all columns (#369) * Check for reserved column names on all columns * Move array --- src/TriggerBinding/SqlTriggerConstants.cs | 11 ++++++++++ src/TriggerBinding/SqlTriggerListener.cs | 25 ++++++++--------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index 8c627202d..8353eb7c1 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -15,6 +15,17 @@ internal static class SqlTriggerConstants public const string LeasesTableAttemptCountColumnName = "_az_func_AttemptCount"; public const string LeasesTableLeaseExpirationTimeColumnName = "_az_func_LeaseExpirationTime"; + /// + /// The column names that are used in internal state tables and so can't exist in the target table + /// since that shares column names with the primary keys from each user table being monitored. + /// + public static readonly string[] ReservedColumnNames = new string[] + { + LeasesTableChangeVersionColumnName, + LeasesTableAttemptCountColumnName, + LeasesTableLeaseExpirationTimeColumnName + }; + public const string ConfigKey_SqlTrigger_BatchSize = "Sql_Trigger_BatchSize"; public const string ConfigKey_SqlTrigger_PollingInterval = "Sql_Trigger_PollingIntervalMs"; } diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index d1d95bb7d..e426f6f7a 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -257,22 +257,6 @@ FROM sys.indexes AS i throw new InvalidOperationException($"Could not find primary key created in table: '{this._userTable.FullName}'."); } - string[] reservedColumnNames = new[] - { - SqlTriggerConstants.LeasesTableChangeVersionColumnName, - SqlTriggerConstants.LeasesTableAttemptCountColumnName, - SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName - }; - - var conflictingColumnNames = primaryKeyColumns.Select(col => col.name).Intersect(reservedColumnNames).ToList(); - - if (conflictingColumnNames.Count > 0) - { - string columnNames = string.Join(", ", conflictingColumnNames.Select(col => $"'{col}'")); - throw new InvalidOperationException($"Found reserved column name(s): {columnNames} in table: '{this._userTable.FullName}'." + - " Please rename them to be able to use trigger binding."); - } - this._logger.LogDebugWithThreadId($"END GetPrimaryKeyColumns ColumnNames(types) = {string.Join(", ", primaryKeyColumns.Select(col => $"'{col.name}({col.type})'"))}."); return primaryKeyColumns; } @@ -317,6 +301,15 @@ FROM sys.columns AS c throw new InvalidOperationException($"Found column(s) with unsupported type(s): {columnNamesAndTypes} in table: '{this._userTable.FullName}'."); } + var conflictingColumnNames = userTableColumns.Intersect(SqlTriggerConstants.ReservedColumnNames).ToList(); + + if (conflictingColumnNames.Count > 0) + { + string columnNames = string.Join(", ", conflictingColumnNames.Select(col => $"'{col}'")); + throw new InvalidOperationException($"Found reserved column name(s): {columnNames} in table: '{this._userTable.FullName}'." + + " Please rename them to be able to use trigger binding."); + } + this._logger.LogDebugWithThreadId($"END GetUserTableColumns ColumnNames = {string.Join(", ", userTableColumns.Select(col => $"'{col}'"))}."); return userTableColumns; } From cec3504acca5f82c905160c54c905bebda8725e8 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 27 Sep 2022 09:44:18 -0700 Subject: [PATCH 36/77] Generate timeout during each test (#371) * Generate timeout during each test * fix count * Fix & increase timeout * Have minimum timeout * Math right... --- src/TriggerBinding/SqlTableChangeMonitor.cs | 11 +- .../SqlTriggerBindingIntegrationTests.cs | 203 ++++++++++-------- 2 files changed, 125 insertions(+), 89 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 3d161a708..d5646d3e0 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -43,6 +43,9 @@ internal sealed class SqlTableChangeMonitor : IDisposable private const int LeaseIntervalInSeconds = 60; private const int LeaseRenewalIntervalInSeconds = 15; private const int MaxRetryReleaseLeases = 3; + + public const int DefaultBatchSize = 10; + public const int DefaultPollingIntervalMs = 5000; #endregion Constants private readonly string _connectionString; @@ -58,11 +61,11 @@ internal sealed class SqlTableChangeMonitor : IDisposable /// /// Number of changes to process in each iteration of the loop /// - private readonly int _batchSize = 10; + private readonly int _batchSize = DefaultBatchSize; /// /// Delay in ms between processing each batch of changes /// - private readonly int _pollingIntervalInMs = 5000; + private readonly int _pollingIntervalInMs = DefaultPollingIntervalMs; private readonly CancellationTokenSource _cancellationTokenSourceCheckForChanges = new CancellationTokenSource(); private readonly CancellationTokenSource _cancellationTokenSourceRenewLeases = new CancellationTokenSource(); @@ -591,8 +594,8 @@ private long RecomputeLastSyncVersion() // have leases acquired on them by another worker. // Therefore, if there are more than one version numbers in the set, return the second highest one. Otherwise, return // the only version number in the set. - // Also this LastSyncVersion is actually updated in the GlobalState table only after verifying that the changes with - // changeVersion <= newLastSyncVersion have been processed in BuildUpdateTablesPostInvocation query. + // Also this LastSyncVersion is actually updated in the GlobalState table only after verifying that the changes with + // changeVersion <= newLastSyncVersion have been processed in BuildUpdateTablesPostInvocation query. long lastSyncVersion = changeVersionSet.ElementAt(changeVersionSet.Count > 1 ? changeVersionSet.Count - 2 : 0); this._logger.LogDebugWithThreadId($"RecomputeLastSyncVersion. LastSyncVersion={lastSyncVersion} ChangeVersionSet={string.Join(",", changeVersionSet)}"); return lastSyncVersion; diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 01440c41a..827bd8abf 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -32,38 +32,41 @@ public async Task SingleOperationTriggerTest() this.EnableChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); - // Considering the polling interval of 5 seconds and batch-size of 10, it should take around 15 seconds to - // process 30 insert operations. An extra 5sec is added as a buffer to the timeout. - // Similar reasoning is used to set delays for update and delete operations. + int firstId = 1; + int lastId = 30; await this.WaitForProductChanges( - 1, - 30, + firstId, + lastId, SqlChangeOperation.Insert, - () => { this.InsertProducts(1, 30); return Task.CompletedTask; }, + () => { this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - 20000); + GetBatchProcessingTimeout(firstId, lastId)); + firstId = 1; + lastId = 20; // All table columns (not just the columns that were updated) would be returned for update operation. await this.WaitForProductChanges( - 1, - 20, + firstId, + lastId, SqlChangeOperation.Update, - () => { this.UpdateProducts(1, 20); return Task.CompletedTask; }, + () => { this.UpdateProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Updated Product {id}", id => id * 100, - 20000); + GetBatchProcessingTimeout(firstId, lastId)); + firstId = 11; + lastId = 30; // The properties corresponding to non-primary key columns would be set to the C# type's default values // (null and 0) for delete operation. await this.WaitForProductChanges( - 11, - 30, + firstId, + lastId, SqlChangeOperation.Delete, - () => { this.DeleteProducts(11, 30); return Task.CompletedTask; }, + () => { this.DeleteProducts(firstId, lastId); return Task.CompletedTask; }, _ => null, _ => 0, - 20000); + GetBatchProcessingTimeout(firstId, lastId)); } /// @@ -72,22 +75,23 @@ await this.WaitForProductChanges( [Fact] public async Task BatchSizeOverrideTriggerTest() { + const int batchSize = 20; + const int firstId = 1; + const int lastId = 40; this.EnableChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTriggerWithValidation), SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { - { "TEST_EXPECTED_BATCH_SIZE", "20" }, - { "Sql_Trigger_BatchSize", "20" } + { "TEST_EXPECTED_BATCH_SIZE", batchSize.ToString() }, + { "Sql_Trigger_BatchSize", batchSize.ToString() } }); - // Considering the polling interval of 5 seconds and batch-size of 20, it should take around 10sec to - // process 40 insert operations. Added buffer time to timeout for total of 15sec. await this.WaitForProductChanges( - 1, - 40, + firstId, + lastId, SqlChangeOperation.Insert, - () => { this.InsertProducts(1, 40); return Task.CompletedTask; }, + () => { this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - 15000); + GetBatchProcessingTimeout(firstId, lastId, batchSize: batchSize)); } /// @@ -96,21 +100,22 @@ await this.WaitForProductChanges( [Fact] public async Task PollingIntervalOverrideTriggerTest() { + const int pollingIntervalMs = 100; + const int firstId = 1; + const int lastId = 50; this.EnableChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTriggerWithValidation), SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { - { "Sql_Trigger_PollingIntervalMs", "100" } + { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } }); - // Considering the polling interval of 100ms and batch-size of 10, it should take around .5 second to - // process 50 insert operations. Added buffer time to timeout for total of 2sec. await this.WaitForProductChanges( - 1, - 50, + firstId, + lastId, SqlChangeOperation.Insert, - () => { this.InsertProducts(1, 50); return Task.CompletedTask; }, + () => { this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - 2000); + GetBatchProcessingTimeout(firstId, lastId, pollingIntervalMs: pollingIntervalMs)); } @@ -121,70 +126,78 @@ await this.WaitForProductChanges( [Fact] public async Task MultiOperationTriggerTest() { + int firstId = 1; + int lastId = 5; this.EnableChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); // 1. Insert + multiple updates to a row are treated as single insert with latest row values. await this.WaitForProductChanges( - 1, - 5, + firstId, + lastId, SqlChangeOperation.Insert, () => { - this.InsertProducts(1, 5); - this.UpdateProducts(1, 5); - this.UpdateProducts(1, 5); + this.InsertProducts(firstId, lastId); + this.UpdateProducts(firstId, lastId); + this.UpdateProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Updated Updated Product {id}", id => id * 100, - 10000); + GetBatchProcessingTimeout(firstId, lastId)); + firstId = 6; + lastId = 10; // 2. Multiple updates to a row are treated as single update with latest row values. // First insert items and wait for those changes to be sent await this.WaitForProductChanges( - 6, - 10, + firstId, + lastId, SqlChangeOperation.Insert, () => { - this.InsertProducts(6, 10); + this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - 10000); + GetBatchProcessingTimeout(firstId, lastId)); + firstId = 6; + lastId = 10; // Now do multiple updates at once and verify the updates are batched together await this.WaitForProductChanges( - 6, - 10, + firstId, + lastId, SqlChangeOperation.Update, () => { - this.UpdateProducts(6, 10); - this.UpdateProducts(6, 10); + this.UpdateProducts(firstId, lastId); + this.UpdateProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Updated Updated Product {id}", id => id * 100, - 10000); + GetBatchProcessingTimeout(firstId, lastId)); + firstId = 11; + lastId = 20; // 3. Insert + (zero or more updates) + delete to a row are treated as single delete with default values for non-primary columns. await this.WaitForProductChanges( - 11, - 20, + firstId, + lastId, SqlChangeOperation.Delete, () => { - this.InsertProducts(11, 20); - this.UpdateProducts(11, 20); - this.DeleteProducts(11, 20); + this.InsertProducts(firstId, lastId); + this.UpdateProducts(firstId, lastId); + this.DeleteProducts(firstId, lastId); return Task.CompletedTask; }, _ => null, _ => 0, - 10000); + GetBatchProcessingTimeout(firstId, lastId)); } @@ -202,15 +215,13 @@ public async Task MultiFunctionTriggerTest() string functionList = $"{nameof(MultiFunctionTrigger.MultiFunctionTrigger1)} {nameof(MultiFunctionTrigger.MultiFunctionTrigger2)}"; this.StartFunctionHost(functionList, SupportedLanguages.CSharp, useTestFolder: true); - // Considering the polling interval of 5 seconds and batch-size of 10, it should take around 15 seconds to - // process 30 insert operations for each trigger-listener. Buffer of 5sec added to timeout. - // Similar reasoning is used to set delays for update and delete operations. - // 1. INSERT + int firstId = 1; + int lastId = 30; // Set up monitoring for Trigger 1... Task changes1Task = this.WaitForProductChanges( - 1, - 30, + firstId, + lastId, SqlChangeOperation.Insert, () => { @@ -218,14 +229,14 @@ public async Task MultiFunctionTriggerTest() }, id => $"Product {id}", id => id * 100, - 20000, + GetBatchProcessingTimeout(firstId, lastId), Trigger1Changes ); // Set up monitoring for Trigger 2... Task changes2Task = this.WaitForProductChanges( - 1, - 30, + firstId, + lastId, SqlChangeOperation.Insert, () => { @@ -233,20 +244,22 @@ public async Task MultiFunctionTriggerTest() }, id => $"Product {id}", id => id * 100, - 25000, + GetBatchProcessingTimeout(firstId, lastId), Trigger2Changes ); // Now that monitoring is set up make the changes and then wait for the monitoring tasks to see them and complete - this.InsertProducts(1, 30); + this.InsertProducts(firstId, lastId); await Task.WhenAll(changes1Task, changes2Task); // 2. UPDATE + firstId = 1; + lastId = 20; // All table columns (not just the columns that were updated) would be returned for update operation. // Set up monitoring for Trigger 1... changes1Task = this.WaitForProductChanges( - 1, - 20, + firstId, + lastId, SqlChangeOperation.Update, () => { @@ -254,13 +267,13 @@ public async Task MultiFunctionTriggerTest() }, id => $"Updated Product {id}", id => id * 100, - 25000, + GetBatchProcessingTimeout(firstId, lastId), Trigger1Changes); // Set up monitoring for Trigger 2... changes2Task = this.WaitForProductChanges( - 1, - 20, + firstId, + lastId, SqlChangeOperation.Update, () => { @@ -268,20 +281,22 @@ public async Task MultiFunctionTriggerTest() }, id => $"Updated Product {id}", id => id * 100, - 25000, + GetBatchProcessingTimeout(firstId, lastId), Trigger2Changes); // Now that monitoring is set up make the changes and then wait for the monitoring tasks to see them and complete - this.UpdateProducts(1, 20); + this.UpdateProducts(firstId, lastId); await Task.WhenAll(changes1Task, changes2Task); // 3. DELETE + firstId = 11; + lastId = 30; // The properties corresponding to non-primary key columns would be set to the C# type's default values // (null and 0) for delete operation. // Set up monitoring for Trigger 1... changes1Task = this.WaitForProductChanges( - 11, - 30, + firstId, + lastId, SqlChangeOperation.Delete, () => { @@ -289,13 +304,13 @@ public async Task MultiFunctionTriggerTest() }, _ => null, _ => 0, - 25000, + GetBatchProcessingTimeout(firstId, lastId), Trigger1Changes); // Set up monitoring for Trigger 2... changes2Task = this.WaitForProductChanges( - 11, - 30, + firstId, + lastId, SqlChangeOperation.Delete, () => { @@ -303,11 +318,11 @@ public async Task MultiFunctionTriggerTest() }, _ => null, _ => 0, - 25000, + GetBatchProcessingTimeout(firstId, lastId), Trigger2Changes); // Now that monitoring is set up make the changes and then wait for the monitoring tasks to see them and complete - this.DeleteProducts(11, 30); + this.DeleteProducts(firstId, lastId); await Task.WhenAll(changes1Task, changes2Task); } @@ -405,29 +420,29 @@ private void MonitorProductChanges(List> changes, string mess }; } - private void InsertProducts(int first_id, int last_id) + private void InsertProducts(int firstId, int lastId) { - int count = last_id - first_id + 1; + int count = lastId - firstId + 1; this.ExecuteNonQuery( "INSERT INTO [dbo].[Products] VALUES\n" + - string.Join(",\n", Enumerable.Range(first_id, count).Select(id => $"({id}, 'Product {id}', {id * 100})")) + ";"); + string.Join(",\n", Enumerable.Range(firstId, count).Select(id => $"({id}, 'Product {id}', {id * 100})")) + ";"); } - private void UpdateProducts(int first_id, int last_id) + private void UpdateProducts(int firstId, int lastId) { - int count = last_id - first_id + 1; + int count = lastId - firstId + 1; this.ExecuteNonQuery( "UPDATE [dbo].[Products]\n" + "SET Name = 'Updated ' + Name\n" + - "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(first_id, count)) + ");"); + "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(firstId, count)) + ");"); } - private void DeleteProducts(int first_id, int last_id) + private void DeleteProducts(int firstId, int lastId) { - int count = last_id - first_id + 1; + int count = lastId - firstId + 1; this.ExecuteNonQuery( "DELETE FROM [dbo].[Products]\n" + - "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(first_id, count)) + ");"); + "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(firstId, count)) + ");"); } private async Task WaitForProductChanges( @@ -519,5 +534,23 @@ void OutputHandler(object sender, DataReceivedEventArgs e) Assert.True(isCompleted, "Functions host did not log failure to start SQL trigger listener within specified time."); Assert.Equal(expectedErrorMessage, errorMessage); } + + /// + /// Gets a timeout value to use when processing the given number of changes, based on the + /// default batch size and polling interval. + /// + /// The first ID in the batch to process + /// The last ID in the batch to process + /// The batch size if different than the default batch size + /// The polling interval in ms if different than the default polling interval + /// + protected static int GetBatchProcessingTimeout(int firstId, int lastId, int batchSize = SqlTableChangeMonitor.DefaultBatchSize, int pollingIntervalMs = SqlTableChangeMonitor.DefaultPollingIntervalMs) + { + int changesToProcess = lastId - firstId + 1; + int calculatedTimeout = (int)(Math.Ceiling((double)changesToProcess / batchSize) // The number of batches to process + * pollingIntervalMs // The length to process each batch + * 2); // Double to add buffer time for processing results + return Math.Max(calculatedTimeout, 2000); // Always have a timeout of at least 2sec to ensure we have time for processing the results + } } } \ No newline at end of file From 3aced0763266a72e6944bfe7b9836ff7a593630c Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 27 Sep 2022 11:55:07 -0700 Subject: [PATCH 37/77] Increase default trigger throughput and add benchmark tests (#374) --- performance/SqlBindingBenchmarks.cs | 1 + performance/SqlTriggerBindingPerformance.cs | 51 +++++++++++++++++++ src/TriggerBinding/SqlTableChangeMonitor.cs | 4 +- .../SqlTriggerBindingIntegrationTests.cs | 12 ++--- 4 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 performance/SqlTriggerBindingPerformance.cs diff --git a/performance/SqlBindingBenchmarks.cs b/performance/SqlBindingBenchmarks.cs index b56563db3..7917e17e2 100644 --- a/performance/SqlBindingBenchmarks.cs +++ b/performance/SqlBindingBenchmarks.cs @@ -11,6 +11,7 @@ public static void Main() { BenchmarkRunner.Run(); BenchmarkRunner.Run(); + BenchmarkRunner.Run(); } } } \ No newline at end of file diff --git a/performance/SqlTriggerBindingPerformance.cs b/performance/SqlTriggerBindingPerformance.cs new file mode 100644 index 000000000..60e3053f5 --- /dev/null +++ b/performance/SqlTriggerBindingPerformance.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance +{ + public class SqlTriggerBindingPerformance : SqlTriggerBindingIntegrationTests + { + [GlobalSetup] + public void GlobalSetup() + { + this.EnableChangeTrackingForTable("Products"); + this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); + } + + [Benchmark] + [Arguments(1)] + [Arguments(10)] + [Arguments(100)] + [Arguments(1000)] + public async Task ProductsTriggerTest(int count) + { + await this.WaitForProductChanges( + 1, + count, + SqlChangeOperation.Insert, + () => { this.InsertProducts(1, count); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + GetBatchProcessingTimeout(1, count)); + } + + [IterationCleanup] + public void IterationCleanup() + { + // Delete all rows in Products table after each iteration + this.ExecuteNonQuery("TRUNCATE TABLE Products"); + } + + [GlobalCleanup] + public void GlobalCleanup() + { + this.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index d5646d3e0..d6dd3fda0 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -44,8 +44,8 @@ internal sealed class SqlTableChangeMonitor : IDisposable private const int LeaseRenewalIntervalInSeconds = 15; private const int MaxRetryReleaseLeases = 3; - public const int DefaultBatchSize = 10; - public const int DefaultPollingIntervalMs = 5000; + public const int DefaultBatchSize = 100; + public const int DefaultPollingIntervalMs = 1000; #endregion Constants private readonly string _connectionString; diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 827bd8abf..ff5285e0c 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -18,7 +18,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration [Collection("IntegrationTests")] public class SqlTriggerBindingIntegrationTests : IntegrationTestBase { - public SqlTriggerBindingIntegrationTests(ITestOutputHelper output) : base(output) + public SqlTriggerBindingIntegrationTests(ITestOutputHelper output = null) : base(output) { this.EnableChangeTrackingForDatabase(); } @@ -398,7 +398,7 @@ ALTER DATABASE [{this.DatabaseName}] "); } - private void EnableChangeTrackingForTable(string tableName) + protected void EnableChangeTrackingForTable(string tableName) { this.ExecuteNonQuery($@" ALTER TABLE [dbo].[{tableName}] @@ -420,7 +420,7 @@ private void MonitorProductChanges(List> changes, string mess }; } - private void InsertProducts(int firstId, int lastId) + protected void InsertProducts(int firstId, int lastId) { int count = lastId - firstId + 1; this.ExecuteNonQuery( @@ -428,7 +428,7 @@ private void InsertProducts(int firstId, int lastId) string.Join(",\n", Enumerable.Range(firstId, count).Select(id => $"({id}, 'Product {id}', {id * 100})")) + ";"); } - private void UpdateProducts(int firstId, int lastId) + protected void UpdateProducts(int firstId, int lastId) { int count = lastId - firstId + 1; this.ExecuteNonQuery( @@ -437,7 +437,7 @@ private void UpdateProducts(int firstId, int lastId) "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(firstId, count)) + ");"); } - private void DeleteProducts(int firstId, int lastId) + protected void DeleteProducts(int firstId, int lastId) { int count = lastId - firstId + 1; this.ExecuteNonQuery( @@ -445,7 +445,7 @@ private void DeleteProducts(int firstId, int lastId) "WHERE ProductId IN (" + string.Join(", ", Enumerable.Range(firstId, count)) + ");"); } - private async Task WaitForProductChanges( + protected async Task WaitForProductChanges( int firstId, int lastId, SqlChangeOperation operation, From 8c01883733d700b11551ff1f956fee8f017df1e9 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 27 Sep 2022 12:30:17 -0700 Subject: [PATCH 38/77] Add TriggerMonitorStart event (#376) --- src/Telemetry/Telemetry.cs | 5 +++++ src/TriggerBinding/SqlTableChangeMonitor.cs | 23 +++++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index 2b3f9b8de..c218b3dc2 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -346,6 +346,7 @@ public enum TelemetryEventName TableInfoCacheMiss, TriggerFunctionEnd, TriggerFunctionStart, + TriggerMonitorStart, UpsertEnd, UpsertStart, } @@ -359,6 +360,8 @@ public enum TelemetryPropertyName ErrorName, ExceptionType, HasIdentityColumn, + HasConfiguredBatchSize, + HasConfiguredPollingInterval, LeasesTableName, QueryType, ServerVersion, @@ -373,6 +376,7 @@ public enum TelemetryMeasureName { AcquireLeasesDurationMs, BatchCount, + BatchSize, CommandDurationMs, CreatedSchemaDurationMs, CreateGlobalStateTableDurationMs, @@ -383,6 +387,7 @@ public enum TelemetryMeasureName GetColumnDefinitionsDurationMs, GetPrimaryKeysDurationMs, InsertGlobalStateTableRowDurationMs, + PollingIntervalMs, ReleaseLeasesDurationMs, RetryAttemptNumber, SetLastSyncVersionDurationMs, diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index d6dd3fda0..b07373f56 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -120,16 +120,31 @@ public SqlTableChangeMonitor( this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); this._userTableId = userTableId; + this._telemetryProps = telemetryProps ?? new Dictionary(); + // Check if there's config settings to override the default batch size/polling interval values - this._batchSize = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize) ?? this._batchSize; - this._pollingIntervalInMs = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_PollingInterval) ?? this._pollingIntervalInMs; + int? configuredBatchSize = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize); + int? configuredPollingInterval = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize); + this._batchSize = configuredBatchSize ?? this._batchSize; + this._pollingIntervalInMs = configuredPollingInterval ?? this._pollingIntervalInMs; + var monitorStartProps = new Dictionary(telemetryProps) + { + { TelemetryPropertyName.HasConfiguredBatchSize, (configuredBatchSize != null).ToString() }, + { TelemetryPropertyName.HasConfiguredPollingInterval, (configuredPollingInterval != null).ToString() }, + }; + TelemetryInstance.TrackEvent( + TelemetryEventName.TriggerMonitorStart, + monitorStartProps, + new Dictionary() { + { TelemetryMeasureName.BatchSize, this._batchSize }, + { TelemetryMeasureName.PollingIntervalMs, this._pollingIntervalInMs } + }); + // Prep search-conditions that will be used besides WHERE clause to match table rows. this._rowMatchConditions = Enumerable.Range(0, this._batchSize) .Select(rowIndex => string.Join(" AND ", this._primaryKeyColumns.Select((col, colIndex) => $"{col.AsBracketQuotedString()} = @{rowIndex}_{colIndex}"))) .ToList(); - this._telemetryProps = telemetryProps ?? new Dictionary(); - #pragma warning disable CS4014 // Queue the below tasks and exit. Do not wait for their completion. _ = Task.Run(() => { From 2b043b8be394886b7a675b700476ba1b74e18ae8 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Wed, 5 Oct 2022 20:07:44 +0530 Subject: [PATCH 39/77] Add test for multiple function hosts (#344) --- performance/SqlTriggerBindingPerformance.cs | 2 +- test/Integration/IntegrationTestBase.cs | 78 ++++++----- .../SqlTriggerBindingIntegrationTests.cs | 122 ++++++++++++------ 3 files changed, 132 insertions(+), 70 deletions(-) diff --git a/performance/SqlTriggerBindingPerformance.cs b/performance/SqlTriggerBindingPerformance.cs index 60e3053f5..43a63f5ea 100644 --- a/performance/SqlTriggerBindingPerformance.cs +++ b/performance/SqlTriggerBindingPerformance.cs @@ -32,7 +32,7 @@ await this.WaitForProductChanges( () => { this.InsertProducts(1, count); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - GetBatchProcessingTimeout(1, count)); + this.GetBatchProcessingTimeout(1, count)); } [IterationCleanup] diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index 95170d4cd..99c958cc9 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -1,23 +1,24 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using Microsoft.Data.SqlClient; -using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; using System; +using System.Collections.Generic; using System.Data.Common; using System.Diagnostics; using System.IO; +using System.Linq; using System.Net.Http; using System.Reflection; using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; +using Microsoft.AspNetCore.WebUtilities; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using Microsoft.Data.SqlClient; using Xunit; using Xunit.Abstractions; -using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; -using Microsoft.AspNetCore.WebUtilities; -using System.Collections.Generic; -using System.Linq; + using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry; namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration @@ -25,9 +26,14 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration public class IntegrationTestBase : IDisposable { /// - /// Host process for Azure Function CLI + /// The first Function Host process that was started. Null if no process has been started yet. + /// + protected Process FunctionHost => this.FunctionHostList.FirstOrDefault(); + + /// + /// Host processes for Azure Function CLI. /// - protected Process FunctionHost { get; private set; } + protected List FunctionHostList { get; } = new List(); /// /// Host process for Azurite local storage emulator. This is required for non-HTTP trigger functions: @@ -184,12 +190,16 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag { throw new FileNotFoundException("Working directory not found at " + workingDirectory); } + + // Use a different port for each new host process, starting with the default port number: 7071. + int port = this.Port + this.FunctionHostList.Count; + var startInfo = new ProcessStartInfo { // The full path to the Functions CLI is required in the ProcessStartInfo because UseShellExecute is set to false. // We cannot both use shell execute and redirect output at the same time: https://docs.microsoft.com//dotnet/api/system.diagnostics.processstartinfo.redirectstandardoutput#remarks FileName = GetFunctionsCoreToolsPath(), - Arguments = $"start --verbose --port {this.Port} --functions {functionName}", + Arguments = $"start --verbose --port {port} --functions {functionName}", WorkingDirectory = workingDirectory, WindowStyle = ProcessWindowStyle.Hidden, RedirectStandardOutput = true, @@ -205,24 +215,26 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag startInfo.EnvironmentVariables[TelemetryOptoutEnvVar] = "1"; this.LogOutput($"Starting {startInfo.FileName} {startInfo.Arguments} in {startInfo.WorkingDirectory}"); - this.FunctionHost = new Process + + var functionHost = new Process { StartInfo = startInfo }; + this.FunctionHostList.Add(functionHost); + // Register all handlers before starting the functions host process. var taskCompletionSource = new TaskCompletionSource(); - this.FunctionHost.OutputDataReceived += this.TestOutputHandler; - this.FunctionHost.OutputDataReceived += SignalStartupHandler; + functionHost.OutputDataReceived += SignalStartupHandler; this.FunctionHost.OutputDataReceived += customOutputHandler; - this.FunctionHost.ErrorDataReceived += this.TestOutputHandler; + functionHost.Start(); + functionHost.OutputDataReceived += this.GetTestOutputHandler(functionHost.Id); + functionHost.ErrorDataReceived += this.GetTestOutputHandler(functionHost.Id); + functionHost.BeginOutputReadLine(); + functionHost.BeginErrorReadLine(); - this.FunctionHost.Start(); - this.FunctionHost.BeginOutputReadLine(); - this.FunctionHost.BeginErrorReadLine(); - - this.LogOutput($"Waiting for Azure Function host to start..."); + this.LogOutput("Waiting for Azure Function host to start..."); const int FunctionHostStartupTimeoutInSeconds = 60; bool isCompleted = taskCompletionSource.Task.Wait(TimeSpan.FromSeconds(FunctionHostStartupTimeoutInSeconds)); @@ -233,7 +245,7 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag const int BufferTimeInSeconds = 5; Task.Delay(TimeSpan.FromSeconds(BufferTimeInSeconds)).Wait(); - this.LogOutput($"Azure Function host started!"); + this.LogOutput("Azure Function host started!"); this.FunctionHost.OutputDataReceived -= SignalStartupHandler; void SignalStartupHandler(object sender, DataReceivedEventArgs e) @@ -293,11 +305,16 @@ private void LogOutput(string output) } } - private void TestOutputHandler(object sender, DataReceivedEventArgs e) + private DataReceivedEventHandler GetTestOutputHandler(int processId) { - if (e != null && !string.IsNullOrEmpty(e.Data)) + return TestOutputHandler; + + void TestOutputHandler(object sender, DataReceivedEventArgs e) { - this.LogOutput(e.Data); + if (!string.IsNullOrEmpty(e.Data)) + { + this.LogOutput($"[{processId}] {e.Data}"); + } } } @@ -377,14 +394,17 @@ public void Dispose() this.LogOutput($"Failed to close connection. Error: {e1.Message}"); } - try - { - this.FunctionHost?.Kill(); - this.FunctionHost?.Dispose(); - } - catch (Exception e2) + foreach (Process functionHost in this.FunctionHostList) { - this.LogOutput($"Failed to stop function host, Error: {e2.Message}"); + try + { + functionHost.Kill(); + functionHost.Dispose(); + } + catch (Exception e2) + { + this.LogOutput($"Failed to stop function host, Error: {e2.Message}"); + } } try diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index ff5285e0c..baca890af 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -41,7 +41,7 @@ await this.WaitForProductChanges( () => { this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId)); + this.GetBatchProcessingTimeout(firstId, lastId)); firstId = 1; lastId = 20; @@ -53,7 +53,7 @@ await this.WaitForProductChanges( () => { this.UpdateProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Updated Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId)); + this.GetBatchProcessingTimeout(firstId, lastId)); firstId = 11; lastId = 30; @@ -66,7 +66,7 @@ await this.WaitForProductChanges( () => { this.DeleteProducts(firstId, lastId); return Task.CompletedTask; }, _ => null, _ => 0, - GetBatchProcessingTimeout(firstId, lastId)); + this.GetBatchProcessingTimeout(firstId, lastId)); } /// @@ -91,7 +91,7 @@ await this.WaitForProductChanges( () => { this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId, batchSize: batchSize)); + this.GetBatchProcessingTimeout(firstId, lastId, batchSize: batchSize)); } /// @@ -115,7 +115,7 @@ await this.WaitForProductChanges( () => { this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId, pollingIntervalMs: pollingIntervalMs)); + this.GetBatchProcessingTimeout(firstId, lastId, pollingIntervalMs: pollingIntervalMs)); } @@ -145,7 +145,7 @@ await this.WaitForProductChanges( }, id => $"Updated Updated Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId)); + this.GetBatchProcessingTimeout(firstId, lastId)); firstId = 6; lastId = 10; @@ -162,7 +162,7 @@ await this.WaitForProductChanges( }, id => $"Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId)); + this.GetBatchProcessingTimeout(firstId, lastId)); firstId = 6; lastId = 10; @@ -179,7 +179,7 @@ await this.WaitForProductChanges( }, id => $"Updated Updated Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId)); + this.GetBatchProcessingTimeout(firstId, lastId)); firstId = 11; lastId = 20; @@ -197,10 +197,9 @@ await this.WaitForProductChanges( }, _ => null, _ => 0, - GetBatchProcessingTimeout(firstId, lastId)); + this.GetBatchProcessingTimeout(firstId, lastId)); } - /// /// Ensures correct functionality with multiple user functions tracking the same table. /// @@ -229,7 +228,7 @@ public async Task MultiFunctionTriggerTest() }, id => $"Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId), + this.GetBatchProcessingTimeout(firstId, lastId), Trigger1Changes ); @@ -244,7 +243,7 @@ public async Task MultiFunctionTriggerTest() }, id => $"Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId), + this.GetBatchProcessingTimeout(firstId, lastId), Trigger2Changes ); @@ -267,7 +266,7 @@ public async Task MultiFunctionTriggerTest() }, id => $"Updated Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId), + this.GetBatchProcessingTimeout(firstId, lastId), Trigger1Changes); // Set up monitoring for Trigger 2... @@ -281,7 +280,7 @@ public async Task MultiFunctionTriggerTest() }, id => $"Updated Product {id}", id => id * 100, - GetBatchProcessingTimeout(firstId, lastId), + this.GetBatchProcessingTimeout(firstId, lastId), Trigger2Changes); // Now that monitoring is set up make the changes and then wait for the monitoring tasks to see them and complete @@ -304,7 +303,7 @@ public async Task MultiFunctionTriggerTest() }, _ => null, _ => 0, - GetBatchProcessingTimeout(firstId, lastId), + this.GetBatchProcessingTimeout(firstId, lastId), Trigger1Changes); // Set up monitoring for Trigger 2... @@ -318,7 +317,7 @@ public async Task MultiFunctionTriggerTest() }, _ => null, _ => 0, - GetBatchProcessingTimeout(firstId, lastId), + this.GetBatchProcessingTimeout(firstId, lastId), Trigger2Changes); // Now that monitoring is set up make the changes and then wait for the monitoring tasks to see them and complete @@ -326,13 +325,63 @@ public async Task MultiFunctionTriggerTest() await Task.WhenAll(changes1Task, changes2Task); } + /// + /// Ensures correct functionality with user functions running across multiple functions host processes. + /// + [Fact] + public async Task MultiHostTriggerTest() + { + this.EnableChangeTrackingForTable("Products"); + + // Prepare three function host processes. + this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); + this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); + this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); + + int firstId = 1; + int lastId = 90; + await this.WaitForProductChanges( + firstId, + lastId, + SqlChangeOperation.Insert, + () => { this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + this.GetBatchProcessingTimeout(firstId, lastId)); + + firstId = 1; + lastId = 60; + // All table columns (not just the columns that were updated) would be returned for update operation. + await this.WaitForProductChanges( + firstId, + lastId, + SqlChangeOperation.Update, + () => { this.UpdateProducts(firstId, lastId); return Task.CompletedTask; }, + id => $"Updated Product {id}", + id => id * 100, + this.GetBatchProcessingTimeout(firstId, lastId)); + + firstId = 31; + lastId = 90; + // The properties corresponding to non-primary key columns would be set to the C# type's default values + // (null and 0) for delete operation. + await this.WaitForProductChanges( + firstId, + lastId, + SqlChangeOperation.Delete, + () => { this.DeleteProducts(firstId, lastId); return Task.CompletedTask; }, + _ => null, + _ => 0, + this.GetBatchProcessingTimeout(firstId, lastId)); + } + /// /// Tests the error message when the user table is not present in the database. /// [Fact] public void TableNotPresentTriggerTest() { - this.StartFunctionsHostAndWaitForError( + this.StartFunctionHostAndWaitForError( nameof(TableNotPresentTrigger), true, "Could not find table: 'dbo.TableNotPresent'."); @@ -344,7 +393,7 @@ public void TableNotPresentTriggerTest() [Fact] public void PrimaryKeyNotCreatedTriggerTest() { - this.StartFunctionsHostAndWaitForError( + this.StartFunctionHostAndWaitForError( nameof(PrimaryKeyNotPresentTrigger), true, "Could not find primary key created in table: 'dbo.ProductsWithoutPrimaryKey'."); @@ -357,7 +406,7 @@ public void PrimaryKeyNotCreatedTriggerTest() [Fact] public void ReservedPrimaryKeyColumnNamesTriggerTest() { - this.StartFunctionsHostAndWaitForError( + this.StartFunctionHostAndWaitForError( nameof(ReservedPrimaryKeyColumnNamesTrigger), true, "Found reserved column name(s): '_az_func_ChangeVersion', '_az_func_AttemptCount', '_az_func_LeaseExpirationTime' in table: 'dbo.ProductsWithReservedPrimaryKeyColumnNames'." + @@ -370,7 +419,7 @@ public void ReservedPrimaryKeyColumnNamesTriggerTest() [Fact] public void UnsupportedColumnTypesTriggerTest() { - this.StartFunctionsHostAndWaitForError( + this.StartFunctionHostAndWaitForError( nameof(UnsupportedColumnTypesTrigger), true, "Found column(s) with unsupported type(s): 'Location' (type: geography), 'Geometry' (type: geometry), 'Organization' (type: hierarchyid)" + @@ -383,7 +432,7 @@ public void UnsupportedColumnTypesTriggerTest() [Fact] public void ChangeTrackingNotEnabledTriggerTest() { - this.StartFunctionsHostAndWaitForError( + this.StartFunctionHostAndWaitForError( nameof(ProductsTrigger), false, "Could not find change tracking enabled for table: 'dbo.Products'."); @@ -406,20 +455,6 @@ ALTER TABLE [dbo].[{tableName}] "); } - private void MonitorProductChanges(List> changes, string messagePrefix) - { - int index = 0; - - this.FunctionHost.OutputDataReceived += (sender, e) => - { - if (e.Data != null && (index = e.Data.IndexOf(messagePrefix, StringComparison.Ordinal)) >= 0) - { - string json = e.Data[(index + messagePrefix.Length)..]; - changes.AddRange(JsonConvert.DeserializeObject>>(json)); - } - }; - } - protected void InsertProducts(int firstId, int lastId) { int count = lastId - firstId + 1; @@ -483,7 +518,10 @@ void MonitorOutputData(object sender, DataReceivedEventArgs e) } }; // Set up listener for the changes coming in - this.FunctionHost.OutputDataReceived += MonitorOutputData; + foreach (Process functionHost in this.FunctionHostList) + { + functionHost.OutputDataReceived += MonitorOutputData; + } // Now that we've set up our listener trigger the actions to monitor await actions(); @@ -492,7 +530,10 @@ void MonitorOutputData(object sender, DataReceivedEventArgs e) await taskCompletion.Task.TimeoutAfter(TimeSpan.FromMilliseconds(timeoutMs), $"Timed out waiting for {operation} changes."); // Unhook handler since we're done monitoring these changes so we aren't checking other changes done later - this.FunctionHost.OutputDataReceived -= MonitorOutputData; + foreach (Process functionHost in this.FunctionHostList) + { + functionHost.OutputDataReceived -= MonitorOutputData; + } } /// @@ -502,7 +543,7 @@ void MonitorOutputData(object sender, DataReceivedEventArgs e) /// Name of the user function that should cause error in trigger listener /// Whether the functions host should be launched from test folder /// Expected error message string - private void StartFunctionsHostAndWaitForError(string functionName, bool useTestFolder, string expectedErrorMessage) + private void StartFunctionHostAndWaitForError(string functionName, bool useTestFolder, string expectedErrorMessage) { string errorMessage = null; var tcs = new TaskCompletionSource(); @@ -544,10 +585,11 @@ void OutputHandler(object sender, DataReceivedEventArgs e) /// The batch size if different than the default batch size /// The polling interval in ms if different than the default polling interval /// - protected static int GetBatchProcessingTimeout(int firstId, int lastId, int batchSize = SqlTableChangeMonitor.DefaultBatchSize, int pollingIntervalMs = SqlTableChangeMonitor.DefaultPollingIntervalMs) + protected int GetBatchProcessingTimeout(int firstId, int lastId, int batchSize = SqlTableChangeMonitor.DefaultBatchSize, int pollingIntervalMs = SqlTableChangeMonitor.DefaultPollingIntervalMs) { int changesToProcess = lastId - firstId + 1; - int calculatedTimeout = (int)(Math.Ceiling((double)changesToProcess / batchSize) // The number of batches to process + int calculatedTimeout = (int)(Math.Ceiling((double)changesToProcess / batchSize // The number of batches to process + / this.FunctionHostList.Count) // The number of function host processes * pollingIntervalMs // The length to process each batch * 2); // Double to add buffer time for processing results return Math.Max(calculatedTimeout, 2000); // Always have a timeout of at least 2sec to ensure we have time for processing the results From ed8b2681deae55a365bc14072c57b9545853dff8 Mon Sep 17 00:00:00 2001 From: AmeyaRele <35621237+AmeyaRele@users.noreply.github.com> Date: Mon, 10 Oct 2022 09:17:12 +0530 Subject: [PATCH 40/77] Add comment for explaining token cancellation (#387) --- src/TriggerBinding/SqlTableChangeMonitor.cs | 1 + src/TriggerBinding/SqlTriggerListener.cs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index b07373f56..e52adc093 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -156,6 +156,7 @@ public SqlTableChangeMonitor( public void Dispose() { + // When the CheckForChanges loop is finished, it will cancel the lease renewal loop. this._cancellationTokenSourceCheckForChanges.Cancel(); } diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index e426f6f7a..bb485f629 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -115,7 +115,6 @@ public async Task StartAsync(CancellationToken cancellationToken) this._logger.LogInformation($"Starting SQL trigger listener for table: '{this._userTable.FullName}', function ID: '{this._userFunctionId}'."); - // TODO: Check if passing the cancellation token would be beneficial. this._changeMonitor = new SqlTableChangeMonitor( this._connectionString, userTableId, From 72751fe14666e836dd106ba628a7f1b8dbb10e9f Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Wed, 12 Oct 2022 10:21:23 -0700 Subject: [PATCH 41/77] Update README and code comments (#391) * Update README and code comments * more * PR comments * Add cleanup docs * Fix spelling * pr comments --- README.md | 31 +++++++++++++++++++++ src/TriggerBinding/SqlTableChangeMonitor.cs | 11 +++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fb6166bac..d3b3dab10 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ Azure SQL bindings for Azure Functions are supported for: - [Primary Keys and Identity Columns](#primary-keys-and-identity-columns) - [Trigger Binding](#trigger-binding) - [Change Tracking](#change-tracking) + - [Internal State Tables](#internal-state-tables) + - [az_func.GlobalState](#az_funcglobalstate) + - [az_func.Leases_*](#az_funcleases_) - [Trigger Samples](#trigger-samples) - [Known Issues](#known-issues) - [Telemetry](#telemetry) @@ -869,6 +872,34 @@ The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sq > **NOTE:** The leases table contains all columns corresponding to the primary key from the user table and three additional columns named `_az_func_ChangeVersion`, `_az_func_AttemptCount` and `_az_func_LeaseExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. +#### Internal State Tables + +The trigger functionality creates several tables to use for tracking the current state of the trigger. This allows state to be persisted across sessions and for multiple instances of a trigger binding to execute in parallel (for scaling purposes). + +In addition, a schema named `az_func` will be created that the tables will belong to. + +The login the trigger is configured to use must be given permissions to create these tables and schema. If not, then an error will be thrown and the trigger will fail to run. + +If the tables are deleted or modified, then unexpected behavior may occur. To reset the state of the triggers, first stop all currently running functions with trigger bindings and then either truncate or delete the tables. The next time a function with a trigger binding is started, it will recreate the tables as necessary. + +##### az_func.GlobalState + +This table stores information about each function being executed, what table that function is watching and what the [last sync state](https://learn.microsoft.com/sql/relational-databases/track-changes/work-with-change-tracking-sql-server) that has been processed. + +##### az_func.Leases_* + +A `Leases_*` table is created for every unique instance of a function and table. The full name will be in the format `Leases__` where `` is generated from the function ID and `` is the object ID of the table being tracked. Such as `Leases_7d12c06c6ddff24c_1845581613`. + +This table is used to ensure that all changes are processed and that no change is processed more than once. This table consists of two groups of columns: + + * A column for each column in the primary key of the target table - used to identify the row that it maps to in the target table + * A couple columns for tracking the state of each row. These are: + * `_az_func_ChangeVersion` for the change version of the row currently being processed + * `_az_func_AttemptCount` for tracking the number of times that a change has attempted to be processed to avoid getting stuck trying to process a change it's unable to handle + * `_az_func_LeaseExpirationTime` for tracking when the lease on this row for a particular instance is set to expire. This ensures that if an instance exits unexpectedly another instance will be able to pick up and process any changes it had leases for after the expiration time has passed. + +A row is created for every row in the target table that is modified. These are then cleaned up after the changes are processed for a set of changes corresponding to a change tracking sync version. + #### Trigger Samples The trigger binding takes two [arguments](https://github.com/Azure/azure-functions-sql-extension/blob/main/src/TriggerBinding/SqlTriggerAttribute.cs) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index e52adc093..5ed030ae3 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -331,14 +331,17 @@ private async Task ProcessTableChangesAsync(SqlConnection connection, Cancellati try { - // What should we do if this fails? It doesn't make sense to retry since it's not a connection based - // thing. We could still try to trigger on the correctly processed changes, but that adds additional - // complication because we don't want to release the leases on the incorrectly processed changes. - // For now, just give up I guess? changes = this.ProcessChanges(); } catch (Exception e) { + // Either there's a bug or we're in a bad state so not much we can do here. We'll try clearing + // our state and retry getting the changes from the top again in case something broke while + // fetching the changes. + // It doesn't make sense to retry processing the changes immediately since this isn't a connection-based issue. + // We could probably send up the changes we were able to process and just skip the ones we couldn't, but given + // that this is not a case we expect would happen during normal execution we'll err on the side of caution for + // now and just retry getting the whole set of changes. this._logger.LogError($"Failed to compose trigger parameter value for table: '{this._userTable.FullName} due to exception: {e.GetType()}. Exception message: {e.Message}"); TelemetryInstance.TrackException(TelemetryErrorName.ProcessChanges, e, this._telemetryProps); await this.ClearRowsAsync(); From 3a4ddd387d0b21f48282c79db3e9e646b94d7b23 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 11:02:53 -0700 Subject: [PATCH 42/77] Fix polling size override using wrong configuration value --- src/TriggerBinding/SqlTableChangeMonitor.cs | 2 +- test/Common/TestUtils.cs | 31 +++++++++++++ .../SqlTriggerBindingIntegrationTests.cs | 43 +++++++++++++++---- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 5ed030ae3..bdfccca79 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -124,7 +124,7 @@ public SqlTableChangeMonitor( // Check if there's config settings to override the default batch size/polling interval values int? configuredBatchSize = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize); - int? configuredPollingInterval = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize); + int? configuredPollingInterval = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_PollingInterval); this._batchSize = configuredBatchSize ?? this._batchSize; this._pollingIntervalInMs = configuredPollingInterval ?? this._pollingIntervalInMs; var monitorStartProps = new Dictionary(telemetryProps) diff --git a/test/Common/TestUtils.cs b/test/Common/TestUtils.cs index ab45410a3..7cef4e51d 100644 --- a/test/Common/TestUtils.cs +++ b/test/Common/TestUtils.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Data; using System.Diagnostics; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -177,5 +178,35 @@ public static async Task TimeoutAfter(this Task task, throw new TimeoutException(message); } } + + /// + /// Creates a DataReceievedEventHandler that will wait for the specified regex and then check that + /// the matched group matches the expected value. + /// + /// The task completion source to signal when the value is receieved + /// The regex. This must have a single group match for the specific value being looked for + /// The name of the value to output if the match fails + /// The value expected to be equal to the matched group from the regex + /// + public static DataReceivedEventHandler CreateOutputReceievedHandler(TaskCompletionSource taskCompletionSource, string regex, string valueName, string expectedValue) + { + return (object sender, DataReceivedEventArgs e) => + { + Match match = Regex.Match(e.Data, regex); + if (match.Success) + { + // We found the line so now check that the group matches our expected value + string actualValue = match.Groups[1].Value; + if (actualValue == expectedValue) + { + taskCompletionSource.SetResult(true); + } + else + { + taskCompletionSource.SetException(new Exception($"Expected {valueName} value of {expectedValue} but got value {actualValue}")); + } + } + }; + } } } diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index baca890af..a8c3ab4a7 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -79,10 +79,22 @@ public async Task BatchSizeOverrideTriggerTest() const int firstId = 1; const int lastId = 40; this.EnableChangeTrackingForTable("Products"); - this.StartFunctionHost(nameof(ProductsTriggerWithValidation), SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { - { "TEST_EXPECTED_BATCH_SIZE", batchSize.ToString() }, - { "Sql_Trigger_BatchSize", batchSize.ToString() } - }); + var taskCompletionSource = new TaskCompletionSource(); + DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( + taskCompletionSource, + @"Starting change consumption loop. BatchSize: \d* PollingIntervalMs: (\d*)", + "PollingInterval", + batchSize.ToString()); + this.StartFunctionHost( + nameof(ProductsTriggerWithValidation), + SupportedLanguages.CSharp, + useTestFolder: true, + customOutputHandler: handler, + environmentVariables: new Dictionary() { + { "TEST_EXPECTED_BATCH_SIZE", batchSize.ToString() }, + { "Sql_Trigger_BatchSize", batchSize.ToString() } + } + ); await this.WaitForProductChanges( firstId, @@ -92,6 +104,7 @@ await this.WaitForProductChanges( id => $"Product {id}", id => id * 100, this.GetBatchProcessingTimeout(firstId, lastId, batchSize: batchSize)); + await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000)); } /// @@ -100,13 +113,25 @@ await this.WaitForProductChanges( [Fact] public async Task PollingIntervalOverrideTriggerTest() { - const int pollingIntervalMs = 100; const int firstId = 1; const int lastId = 50; + const int pollingIntervalMs = 75; this.EnableChangeTrackingForTable("Products"); - this.StartFunctionHost(nameof(ProductsTriggerWithValidation), SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { - { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } - }); + var taskCompletionSource = new TaskCompletionSource(); + DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( + taskCompletionSource, + @"Starting change consumption loop. BatchSize: \d* PollingIntervalMs: (\d*)", + "PollingInterval", + pollingIntervalMs.ToString()); + this.StartFunctionHost( + nameof(ProductsTriggerWithValidation), + SupportedLanguages.CSharp, + useTestFolder: true, + customOutputHandler: handler, + environmentVariables: new Dictionary() { + { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } + } + ); await this.WaitForProductChanges( firstId, @@ -116,9 +141,9 @@ await this.WaitForProductChanges( id => $"Product {id}", id => id * 100, this.GetBatchProcessingTimeout(firstId, lastId, pollingIntervalMs: pollingIntervalMs)); + await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000)); } - /// /// Verifies that if several changes have happened to the table row since last invocation, then a single net /// change for that row is passed to the user function. From 2e0f3454a282d6d67b0f3aed335b366a931434cd Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 11:22:07 -0700 Subject: [PATCH 43/77] Fixes --- .../SqlTriggerBindingIntegrationTests.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index a8c3ab4a7..0579f1527 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -75,15 +75,19 @@ await this.WaitForProductChanges( [Fact] public async Task BatchSizeOverrideTriggerTest() { - const int batchSize = 20; + // Use enough items for the default batch size to require 4 batches but then + // set the batch size to the same value so they can all be processed in one + // batch. The test will only wait for ~1 batch worth of time so will timeout + // if the batch size isn't actually changed + const int batchSize = SqlTableChangeMonitor.DefaultBatchSize * 4; const int firstId = 1; - const int lastId = 40; + const int lastId = batchSize; this.EnableChangeTrackingForTable("Products"); var taskCompletionSource = new TaskCompletionSource(); DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( taskCompletionSource, - @"Starting change consumption loop. BatchSize: \d* PollingIntervalMs: (\d*)", - "PollingInterval", + @"Starting change consumption loop. BatchSize: (\d*) PollingIntervalMs: \d*", + "BatchSize", batchSize.ToString()); this.StartFunctionHost( nameof(ProductsTriggerWithValidation), @@ -114,7 +118,10 @@ await this.WaitForProductChanges( public async Task PollingIntervalOverrideTriggerTest() { const int firstId = 1; - const int lastId = 50; + // Use enough items to require 5 batches to be processed - the test will + // only wait for the expected time and timeout if the default polling + // interval isn't actually modified. + const int lastId = SqlTableChangeMonitor.DefaultBatchSize * 5; const int pollingIntervalMs = 75; this.EnableChangeTrackingForTable("Products"); var taskCompletionSource = new TaskCompletionSource(); From 360587c291c3909db6e3e0577c898b06931c4174 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 12:07:48 -0700 Subject: [PATCH 44/77] Update comments --- test/Common/TestUtils.cs | 2 +- test/Integration/SqlTriggerBindingIntegrationTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Common/TestUtils.cs b/test/Common/TestUtils.cs index 7cef4e51d..ae0d0f657 100644 --- a/test/Common/TestUtils.cs +++ b/test/Common/TestUtils.cs @@ -187,7 +187,7 @@ public static async Task TimeoutAfter(this Task task, /// The regex. This must have a single group match for the specific value being looked for /// The name of the value to output if the match fails /// The value expected to be equal to the matched group from the regex - /// + /// The event handler public static DataReceivedEventHandler CreateOutputReceievedHandler(TaskCompletionSource taskCompletionSource, string regex, string valueName, string expectedValue) { return (object sender, DataReceivedEventArgs e) => diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 0579f1527..7fbb9a81c 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -75,7 +75,7 @@ await this.WaitForProductChanges( [Fact] public async Task BatchSizeOverrideTriggerTest() { - // Use enough items for the default batch size to require 4 batches but then + // Use enough items to require 4 batches to be processed but then // set the batch size to the same value so they can all be processed in one // batch. The test will only wait for ~1 batch worth of time so will timeout // if the batch size isn't actually changed From 42d301e8edde7488c39e7d1a6a33283e6a73006d Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 12:12:32 -0700 Subject: [PATCH 45/77] typo --- test/Common/TestUtils.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Common/TestUtils.cs b/test/Common/TestUtils.cs index ae0d0f657..8ba874b9f 100644 --- a/test/Common/TestUtils.cs +++ b/test/Common/TestUtils.cs @@ -183,7 +183,7 @@ public static async Task TimeoutAfter(this Task task, /// Creates a DataReceievedEventHandler that will wait for the specified regex and then check that /// the matched group matches the expected value. /// - /// The task completion source to signal when the value is receieved + /// The task completion source to signal when the value is received /// The regex. This must have a single group match for the specific value being looked for /// The name of the value to output if the match fails /// The value expected to be equal to the matched group from the regex From 450180e43c9b1514be3786425898963ab89f0947 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 14:26:53 -0700 Subject: [PATCH 46/77] more changes --- test/Integration/SqlTriggerBindingIntegrationTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 7fbb9a81c..a1f6d4c80 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -108,7 +108,7 @@ await this.WaitForProductChanges( id => $"Product {id}", id => id * 100, this.GetBatchProcessingTimeout(firstId, lastId, batchSize: batchSize)); - await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000)); + await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000), "Timed out waiting for BatchSize configuration message"); } /// @@ -122,7 +122,7 @@ public async Task PollingIntervalOverrideTriggerTest() // only wait for the expected time and timeout if the default polling // interval isn't actually modified. const int lastId = SqlTableChangeMonitor.DefaultBatchSize * 5; - const int pollingIntervalMs = 75; + const int pollingIntervalMs = SqlTableChangeMonitor.DefaultPollingIntervalMs / 2; this.EnableChangeTrackingForTable("Products"); var taskCompletionSource = new TaskCompletionSource(); DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( @@ -148,7 +148,7 @@ await this.WaitForProductChanges( id => $"Product {id}", id => id * 100, this.GetBatchProcessingTimeout(firstId, lastId, pollingIntervalMs: pollingIntervalMs)); - await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000)); + await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000), "Timed out waiting for PollingInterval configuration message"); } /// From 513e3ffde3994a6a514eea05c855565cba1dd80b Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Mon, 17 Oct 2022 15:25:51 -0700 Subject: [PATCH 47/77] Use merge query for renewing leases --- src/TriggerBinding/SqlTableChangeMonitor.cs | 80 +++++++++++++-------- src/TriggerBinding/SqlTriggerListener.cs | 2 +- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index bdfccca79..5c25b1d15 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -17,6 +17,8 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Microsoft.Extensions.Configuration; +using System.Data; +using MoreLinq; namespace Microsoft.Azure.WebJobs.Extensions.Sql { @@ -54,7 +56,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable private readonly string _userFunctionId; private readonly string _leasesTableName; private readonly IReadOnlyList _userTableColumns; - private readonly IReadOnlyList _primaryKeyColumns; + private readonly IReadOnlyList<(string name, string type)> _primaryKeyColumns; private readonly IReadOnlyList _rowMatchConditions; private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; @@ -104,7 +106,7 @@ public SqlTableChangeMonitor( string userFunctionId, string leasesTableName, IReadOnlyList userTableColumns, - IReadOnlyList primaryKeyColumns, + IReadOnlyList<(string name, string type)> primaryKeyColumns, ITriggeredFunctionExecutor executor, ILogger logger, IConfiguration configuration, @@ -142,7 +144,7 @@ public SqlTableChangeMonitor( // Prep search-conditions that will be used besides WHERE clause to match table rows. this._rowMatchConditions = Enumerable.Range(0, this._batchSize) - .Select(rowIndex => string.Join(" AND ", this._primaryKeyColumns.Select((col, colIndex) => $"{col.AsBracketQuotedString()} = @{rowIndex}_{colIndex}"))) + .Select(rowIndex => string.Join(" AND ", this._primaryKeyColumns.Select((col, colIndex) => $"{col.name.AsBracketQuotedString()} = @{rowIndex}_{colIndex}"))) .ToList(); #pragma warning disable CS4014 // Queue the below tasks and exit. Do not wait for their completion. @@ -230,7 +232,7 @@ private async Task GetTableChangesAsync(SqlConnection connection, CancellationTo var transactionSw = Stopwatch.StartNew(); long setLastSyncVersionDurationMs = 0L, getChangesDurationMs = 0L, acquireLeasesDurationMs = 0L; - using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + using (SqlTransaction transaction = connection.BeginTransaction(IsolationLevel.RepeatableRead)) { try { @@ -523,7 +525,7 @@ private async Task ReleaseLeasesAsync(SqlConnection connection, CancellationToke var transactionSw = Stopwatch.StartNew(); long releaseLeasesDurationMs = 0L, updateLastSyncVersionDurationMs = 0L; - using (SqlTransaction transaction = connection.BeginTransaction(System.Data.IsolationLevel.RepeatableRead)) + using (SqlTransaction transaction = connection.BeginTransaction(IsolationLevel.RepeatableRead)) { try { @@ -637,7 +639,7 @@ private IReadOnlyList> ProcessChanges() // If the row has been deleted, there is no longer any data for it in the user table. The best we can do // is populate the row-item with the primary key values of the row. Dictionary item = operation == SqlChangeOperation.Delete - ? this._primaryKeyColumns.ToDictionary(col => col, col => row[col]) + ? this._primaryKeyColumns.ToDictionary(col => col.name, col => row[col.name]) : this._userTableColumns.ToDictionary(col => col, col => row[col]); changes.Add(new SqlChange(operation, JsonConvert.DeserializeObject(JsonConvert.SerializeObject(item)))); @@ -699,9 +701,9 @@ IF @last_sync_version < @min_valid_version /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransaction transaction) { - string selectList = string.Join(", ", this._userTableColumns.Select(col => this._primaryKeyColumns.Contains(col) ? $"c.{col.AsBracketQuotedString()}" : $"u.{col.AsBracketQuotedString()}")); - string userTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = u.{col.AsBracketQuotedString()}")); - string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = l.{col.AsBracketQuotedString()}")); + string selectList = string.Join(", ", this._userTableColumns.Select(col => this._primaryKeyColumns.Select(c => c.name).Contains(col) ? $"c.{col.AsBracketQuotedString()}" : $"u.{col.AsBracketQuotedString()}")); + string userTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.name.AsBracketQuotedString()} = u.{col.name.AsBracketQuotedString()}")); + string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.name.AsBracketQuotedString()} = l.{col.name.AsBracketQuotedString()}")); string getChangesQuery = $@" DECLARE @last_sync_version bigint; @@ -737,28 +739,46 @@ LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCo /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransaction transaction, IReadOnlyList> rows) { - var acquireLeasesQuery = new StringBuilder(); - - for (int rowIndex = 0; rowIndex < rows.Count; rowIndex++) + // The column definitions to use for the CTE + IEnumerable cteColumnDefinitions = this._primaryKeyColumns + .Select(c => $"{c.name.AsBracketQuotedString()} {c.type}") + // These are the internal column values that we use. Note that we use SYS_CHANGE_VERSION because that's + // the new version - the _az_func_ChangeVersion has the old version + .Concat(new string[] { "SYS_CHANGE_VERSION bigint", "_az_func_AttemptCount int" }); + IList bracketedPrimaryKeys = this._primaryKeyColumns.Select(p => p.name.AsBracketQuotedString()).ToList(); + + // Create the query that the merge statement will match the rows on + var primaryKeyMatchingQuery = new StringBuilder($"ExistingData.{bracketedPrimaryKeys[0]} = NewData.{bracketedPrimaryKeys[0]}"); + foreach (string primaryKey in bracketedPrimaryKeys.Skip(1)) { - string valuesList = string.Join(", ", this._primaryKeyColumns.Select((_, colIndex) => $"@{rowIndex}_{colIndex}")); - string changeVersion = rows[rowIndex]["SYS_CHANGE_VERSION"].ToString(); - - acquireLeasesQuery.Append($@" - IF NOT EXISTS (SELECT * FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}) - INSERT INTO {this._leasesTableName} WITH (TABLOCKX) - VALUES ({valuesList}, {changeVersion}, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME())); - ELSE - UPDATE {this._leasesTableName} WITH (TABLOCKX) - SET - {SqlTriggerConstants.LeasesTableChangeVersionColumnName} = {changeVersion}, - {SqlTriggerConstants.LeasesTableAttemptCountColumnName} = {SqlTriggerConstants.LeasesTableAttemptCountColumnName} + 1, - {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) - WHERE {this._rowMatchConditions[rowIndex]}; - "); + primaryKeyMatchingQuery.Append($" AND ExistingData.{primaryKey} = NewData.{primaryKey}"); } - return this.GetSqlCommandWithParameters(acquireLeasesQuery.ToString(), connection, transaction, rows); + const string acquireLeasesCte = "acquireLeasesCte"; + const string rowDataParameter = "@rowData"; + // Create the merge query that will either update the rows that already exist or insert a new one if it doesn't exist + string query = $@" + WITH {acquireLeasesCte} AS ( SELECT * FROM OPENJSON(@rowData) WITH ({string.Join(",", cteColumnDefinitions)}) ) + MERGE INTO {this._leasesTableName} WITH (TABLOCKX) + AS ExistingData + USING {acquireLeasesCte} + AS NewData + ON + {primaryKeyMatchingQuery} + WHEN MATCHED THEN + UPDATE SET + {SqlTriggerConstants.LeasesTableChangeVersionColumnName} = NewData.SYS_CHANGE_VERSION, + {SqlTriggerConstants.LeasesTableAttemptCountColumnName} = ExistingData.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} + 1, + {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + WHEN NOT MATCHED THEN + INSERT VALUES ({string.Join(",", bracketedPrimaryKeys.Select(k => $"NewData.{k}"))}, NewData.SYS_CHANGE_VERSION, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()));"; + + var command = new SqlCommand(query, connection, transaction); + SqlParameter par = command.Parameters.Add(rowDataParameter, SqlDbType.NVarChar, -1); + string rowData = JsonConvert.SerializeObject(rows); + this._logger.LogInformation(rowData); + par.Value = rowData; + return command; } /// @@ -823,7 +843,7 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, SqlTransaction transaction, long newLastSyncVersion) { - string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = l.{col.AsBracketQuotedString()}")); + string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.name.AsBracketQuotedString()} = l.{col.name.AsBracketQuotedString()}")); string updateTablesPostInvocationQuery = $@" DECLARE @current_last_sync_version bigint; @@ -879,7 +899,7 @@ private SqlCommand GetSqlCommandWithParameters(string commandText, SqlConnection var command = new SqlCommand(commandText, connection, transaction); SqlParameter[] parameters = Enumerable.Range(0, rows.Count) - .SelectMany(rowIndex => this._primaryKeyColumns.Select((col, colIndex) => new SqlParameter($"@{rowIndex}_{colIndex}", rows[rowIndex][col]))) + .SelectMany(rowIndex => this._primaryKeyColumns.Select((col, colIndex) => new SqlParameter($"@{rowIndex}_{colIndex}", rows[rowIndex][col.name]))) .ToArray(); command.Parameters.AddRange(parameters); diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index bb485f629..f2d4121d5 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -122,7 +122,7 @@ public async Task StartAsync(CancellationToken cancellationToken) this._userFunctionId, leasesTableName, userTableColumns, - primaryKeyColumns.Select(col => col.name).ToList(), + primaryKeyColumns, this._executor, this._logger, this._configuration, From 1dc05122db07cc73b194c1f53f525c67fd60753c Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Tue, 18 Oct 2022 22:18:23 +0530 Subject: [PATCH 48/77] Add runtime driven scaling support (#389) --- README.md | 7 + src/Telemetry/Telemetry.cs | 8 + src/TriggerBinding/SqlTableChangeMonitor.cs | 69 +++++++++ src/TriggerBinding/SqlTriggerListener.cs | 158 +++++++++++++++++++- src/TriggerBinding/SqlTriggerMetrics.cs | 15 ++ 5 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 src/TriggerBinding/SqlTriggerMetrics.cs diff --git a/README.md b/README.md index 9c0c7b966..c81ab170f 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Azure SQL bindings for Azure Functions are supported for: - [Columns with Default Values](#columns-with-default-values) - [Trigger Binding](#trigger-binding) - [Change Tracking](#change-tracking) + - [Scaling](#scaling) - [Internal State Tables](#internal-state-tables) - [az_func.GlobalState](#az_funcglobalstate) - [az_func.Leases_*](#az_funcleases_) @@ -880,6 +881,12 @@ The trigger binding utilizes SQL [change tracking](https://docs.microsoft.com/sq > **NOTE:** The leases table contains all columns corresponding to the primary key from the user table and three additional columns named `_az_func_ChangeVersion`, `_az_func_AttemptCount` and `_az_func_LeaseExpirationTime`. If any of the primary key columns happen to have the same name, that will result in an error message listing any conflicts. In this case, the listed primary key columns must be renamed for the trigger to work. +#### Scaling + +If your application containing functions with SQL trigger bindings is running as an Azure function app, it will be scaled automatically based on the amount of changes that are pending to be processed in the user table. As of today, we only support scaling of function apps running in Elastic Premium plan. To enable scaling, you will need to go the function app resource's page on Azure Portal, then to Configuration > 'Function runtime settings' and turn on 'Runtime Scale Monitoring'. For more information, check documentation on [Runtime Scaling](https://learn.microsoft.com/azure/azure-functions/event-driven-scaling#runtime-scaling). You can configure scaling parameters by going to 'Scale out (App Service plan)' setting on the function app's page. To understand various scale settings, please check the respective sections in [Azure Functions Premium plan](https://learn.microsoft.com/azure/azure-functions/functions-premium-plan?tabs=portal#eliminate-cold-starts)'s documentation. + +There are a couple of checks made to decide on whether the host application needs to be scaled in or out. The rationale behind these checks is to ensure that the count of pending changes per application-worker stays below a certain maximum limit, which is defaulted to 1000, while also ensuring that the number of workers running stays minimal. The scaling decision is made based on the latest count of the pending changes and whether the last 5 times we checked the count, we found it to be continuously increasing or decreasing. + #### Internal State Tables The trigger functionality creates several tables to use for tracking the current state of the trigger. This allows state to be persisted across sessions and for multiple instances of a trigger binding to execute in parallel (for scaling purposes). diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index c218b3dc2..cc0aa9313 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -332,6 +332,7 @@ public enum TelemetryEventName GetChangesStart, GetColumnDefinitions, GetPrimaryKeys, + GetScaleStatus, GetTableInfoEnd, GetTableInfoStart, ReleaseLeasesEnd, @@ -364,9 +365,12 @@ public enum TelemetryPropertyName HasConfiguredPollingInterval, LeasesTableName, QueryType, + ScaleRecommendation, ServerVersion, + TriggerMetrics, Type, UserFunctionId, + WorkerCount, } /// @@ -386,12 +390,14 @@ public enum TelemetryMeasureName GetChangesDurationMs, GetColumnDefinitionsDurationMs, GetPrimaryKeysDurationMs, + GetUnprocessedChangesDurationMs, InsertGlobalStateTableRowDurationMs, PollingIntervalMs, ReleaseLeasesDurationMs, RetryAttemptNumber, SetLastSyncVersionDurationMs, TransactionDurationMs, + UnprocessedChangeCount, UpdateLastSyncVersionDurationMs, } @@ -409,6 +415,8 @@ public enum TelemetryErrorName GetColumnDefinitions, GetColumnDefinitionsTableDoesNotExist, GetPrimaryKeys, + GetScaleStatus, + GetUnprocessedChangeCount, MissingPrimaryKeys, NoPrimaryKeys, ProcessChanges, diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index bdfccca79..2a697b1e5 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -160,6 +160,46 @@ public void Dispose() this._cancellationTokenSourceCheckForChanges.Cancel(); } + public async Task GetUnprocessedChangeCountAsync() + { + long unprocessedChangeCount = 0L; + + try + { + long getUnprocessedChangesDurationMs = 0L; + + using (var connection = new SqlConnection(this._connectionString)) + { + this._logger.LogDebugWithThreadId("BEGIN OpenGetUnprocessedChangesConnection"); + await connection.OpenAsync(); + this._logger.LogDebugWithThreadId("END OpenGetUnprocessedChangesConnection"); + + using (SqlCommand getUnprocessedChangesCommand = this.BuildGetUnprocessedChangesCommand(connection)) + { + this._logger.LogDebugWithThreadId($"BEGIN GetUnprocessedChangeCount Query={getUnprocessedChangesCommand.CommandText}"); + var commandSw = Stopwatch.StartNew(); + unprocessedChangeCount = (long)await getUnprocessedChangesCommand.ExecuteScalarAsync(); + getUnprocessedChangesDurationMs = commandSw.ElapsedMilliseconds; + } + + this._logger.LogDebugWithThreadId($"END GetUnprocessedChangeCount Duration={getUnprocessedChangesDurationMs}ms Count={unprocessedChangeCount}"); + } + + var measures = new Dictionary + { + [TelemetryMeasureName.GetUnprocessedChangesDurationMs] = getUnprocessedChangesDurationMs, + [TelemetryMeasureName.UnprocessedChangeCount] = unprocessedChangeCount, + }; + } + catch (Exception ex) + { + this._logger.LogError($"Failed to query count of unprocessed changes for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.GetUnprocessedChangeCount, ex, this._telemetryProps); + } + + return unprocessedChangeCount; + } + /// /// Executed once every period. If the state of the change monitor is /// , then the method query the change/leases tables for changes on the @@ -727,6 +767,35 @@ LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCo return new SqlCommand(getChangesQuery, connection, transaction); } + /// + /// Builds the query to get count of unprocessed changes in the user's table. This one mimics the query that is + /// used by workers to get the changes for processing. + /// + /// The connection to add to the returned SqlCommand + /// The SqlCommand populated with the query and appropriate parameters + private SqlCommand BuildGetUnprocessedChangesCommand(SqlConnection connection) + { + string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = l.{col.AsBracketQuotedString()}")); + + string getUnprocessedChangesQuery = $@" + DECLARE @last_sync_version bigint; + SELECT @last_sync_version = LastSyncVersion + FROM {SqlTriggerConstants.GlobalStateTableName} + WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; + + SELECT COUNT_BIG(*) + FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c + LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} + WHERE + (l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NULL AND + (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.SYS_CHANGE_VERSION) OR + l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND + (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}); + "; + + return new SqlCommand(getUnprocessedChangesQuery, connection); + } + /// /// Builds the query to acquire leases on the rows in "_rows" if changes are detected in the user's table /// (). diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index bb485f629..63879cc0d 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -12,9 +12,11 @@ using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry; using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Listeners; +using Microsoft.Azure.WebJobs.Host.Scale; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Configuration; +using MoreLinq; namespace Microsoft.Azure.WebJobs.Extensions.Sql { @@ -22,7 +24,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql /// Represents the listener to SQL table changes. /// /// POCO class representing the row in the user table - internal sealed class SqlTriggerListener : IListener + internal sealed class SqlTriggerListener : IListener, IScaleMonitor { private const int ListenerNotStarted = 0; private const int ListenerStarting = 1; @@ -36,12 +38,15 @@ internal sealed class SqlTriggerListener : IListener private readonly ITriggeredFunctionExecutor _executor; private readonly ILogger _logger; private readonly IConfiguration _configuration; + private readonly ScaleMonitorDescriptor _scaleMonitorDescriptor; private readonly IDictionary _telemetryProps = new Dictionary(); private SqlTableChangeMonitor _changeMonitor; private int _listenerState = ListenerNotStarted; + ScaleMonitorDescriptor IScaleMonitor.Descriptor => this._scaleMonitorDescriptor; + /// /// Initializes a new instance of the class. /// @@ -59,6 +64,8 @@ public SqlTriggerListener(string connectionString, string tableName, string user this._executor = executor ?? throw new ArgumentNullException(nameof(executor)); this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + + this._scaleMonitorDescriptor = new ScaleMonitorDescriptor($"{userFunctionId}-SqlTrigger-{tableName}".ToLower(CultureInfo.InvariantCulture)); } public void Cancel() @@ -463,6 +470,155 @@ PRIMARY KEY ({primaryKeys}) } } + async Task IScaleMonitor.GetMetricsAsync() + { + return await this.GetMetricsAsync(); + } + + public async Task GetMetricsAsync() + { + Debug.Assert(!(this._changeMonitor is null)); + + return new SqlTriggerMetrics + { + UnprocessedChangeCount = await this._changeMonitor.GetUnprocessedChangeCountAsync(), + Timestamp = DateTime.UtcNow, + }; + } + + ScaleStatus IScaleMonitor.GetScaleStatus(ScaleStatusContext context) + { + return this.GetScaleStatusWithTelemetry(context.WorkerCount, context.Metrics?.Cast().ToArray()); + } + + public ScaleStatus GetScaleStatus(ScaleStatusContext context) + { + return this.GetScaleStatusWithTelemetry(context.WorkerCount, context.Metrics?.ToArray()); + } + + private ScaleStatus GetScaleStatusWithTelemetry(int workerCount, SqlTriggerMetrics[] metrics) + { + var status = new ScaleStatus + { + Vote = ScaleVote.None, + }; + + var properties = new Dictionary(this._telemetryProps) + { + [TelemetryPropertyName.ScaleRecommendation] = $"{status.Vote}", + [TelemetryPropertyName.TriggerMetrics] = metrics is null ? "null" : $"[{string.Join(", ", metrics.Select(metric => metric.UnprocessedChangeCount))}]", + [TelemetryPropertyName.WorkerCount] = $"{workerCount}", + }; + + try + { + status = this.GetScaleStatusCore(workerCount, metrics); + + properties[TelemetryPropertyName.ScaleRecommendation] = $"{status.Vote}"; + TelemetryInstance.TrackEvent(TelemetryEventName.GetScaleStatus, properties); + } + catch (Exception ex) + { + this._logger.LogError($"Failed to get scale status for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.GetScaleStatus, ex, properties); + } + + return status; + } + + /// + /// Returns scale recommendation i.e. whether to scale in or out the host application. The recommendation is + /// made based on both the latest metrics and the trend of increase or decrease in the count of unprocessed + /// changes in the user table. In all of the calculations, it is attempted to keep the number of workers minimum + /// while also ensuring that the count of unprocessed changes per worker stays under the maximum limit. + /// + /// The current worker count for the host application. + /// The collection of metrics samples to make the scale decision. + /// + private ScaleStatus GetScaleStatusCore(int workerCount, SqlTriggerMetrics[] metrics) + { + // We require minimum 5 samples to estimate the trend of variation in count of unprocessed changes with + // certain reliability. These samples roughly cover the timespan of past 40 seconds. + const int minSamplesForScaling = 5; + + // Please ensure the Readme file and other public documentation are also updated if this value ever needs to + // be changed. + const int maxChangesPerWorker = 1000; + + var status = new ScaleStatus + { + Vote = ScaleVote.None, + }; + + // Do not make a scale decision unless we have enough samples. + if (metrics is null || (metrics.Length < minSamplesForScaling)) + { + this._logger.LogInformation($"Requesting no-scaling: Insufficient metrics for making scale decision for table: '{this._userTable.FullName}'."); + return status; + } + + string counts = string.Join(" ,", metrics.TakeLast(minSamplesForScaling).Select(metric => metric.UnprocessedChangeCount)); + this._logger.LogInformation($"Unprocessed change counts: [{counts}], worker count: {workerCount}, maximum changes per worker: {maxChangesPerWorker}."); + + // Add worker if the count of unprocessed changes per worker exceeds the maximum limit. + long lastUnprocessedChangeCount = metrics.Last().UnprocessedChangeCount; + if (lastUnprocessedChangeCount > workerCount * maxChangesPerWorker) + { + status.Vote = ScaleVote.ScaleOut; + this._logger.LogInformation($"Requesting scale-out: Found too many unprocessed changes for table: '{this._userTable.FullName}' relative to the number of workers."); + return status; + } + + // Check if there is a continuous increase or decrease in count of unprocessed changes. + bool isIncreasing = true; + bool isDecreasing = true; + for (int index = metrics.Length - minSamplesForScaling; index < metrics.Length - 1; index++) + { + isIncreasing = isIncreasing && metrics[index].UnprocessedChangeCount < metrics[index + 1].UnprocessedChangeCount; + isDecreasing = isDecreasing && (metrics[index].UnprocessedChangeCount == 0 || metrics[index].UnprocessedChangeCount > metrics[index + 1].UnprocessedChangeCount); + } + + if (isIncreasing) + { + // Scale out only if the expected count of unprocessed changes would exceed the combined limit after 30 seconds. + DateTime referenceTime = metrics[metrics.Length - 1].Timestamp - TimeSpan.FromSeconds(30); + SqlTriggerMetrics referenceMetric = metrics.First(metric => metric.Timestamp > referenceTime); + long expectedUnprocessedChangeCount = (2 * metrics[metrics.Length - 1].UnprocessedChangeCount) - referenceMetric.UnprocessedChangeCount; + + if (expectedUnprocessedChangeCount > workerCount * maxChangesPerWorker) + { + status.Vote = ScaleVote.ScaleOut; + this._logger.LogInformation($"Requesting scale-out: Found the unprocessed changes for table: '{this._userTable.FullName}' to be continuously increasing" + + " and may exceed the maximum limit set for the workers."); + return status; + } + else + { + this._logger.LogDebug($"Avoiding scale-out: Found the unprocessed changes for table: '{this._userTable.FullName}' to be increasing" + + " but they may not exceed the maximum limit set for the workers."); + } + } + + if (isDecreasing) + { + // Scale in only if the count of unprocessed changes will not exceed the combined limit post the scale-in operation. + if (lastUnprocessedChangeCount <= (workerCount - 1) * maxChangesPerWorker) + { + status.Vote = ScaleVote.ScaleIn; + this._logger.LogInformation($"Requesting scale-in: Found table: '{this._userTable.FullName}' to be either idle or the unprocessed changes to be continuously decreasing."); + return status; + } + else + { + this._logger.LogDebug($"Avoiding scale-in: Found the unprocessed changes for table: '{this._userTable.FullName}' to be decreasing" + + " but they are high enough to require all existing workers for processing."); + } + } + + this._logger.LogInformation($"Requesting no-scaling: Found the number of unprocessed changes for table: '{this._userTable.FullName}' to not require scaling."); + return status; + } + /// /// Clears the current telemetry property dictionary and initializes the default initial properties. /// diff --git a/src/TriggerBinding/SqlTriggerMetrics.cs b/src/TriggerBinding/SqlTriggerMetrics.cs new file mode 100644 index 000000000..525e5e574 --- /dev/null +++ b/src/TriggerBinding/SqlTriggerMetrics.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.WebJobs.Host.Scale; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + internal sealed class SqlTriggerMetrics : ScaleMetrics + { + /// + /// The number of row changes in the user table that are not yet processed. + /// + public long UnprocessedChangeCount { get; set; } + } +} \ No newline at end of file From eecf69c9300ea34838dbb8dc6906500eb626775f Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 18 Oct 2022 10:19:15 -0700 Subject: [PATCH 49/77] Remove logging --- src/TriggerBinding/SqlTableChangeMonitor.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 5c25b1d15..01618a411 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -776,7 +776,6 @@ WHEN NOT MATCHED THEN var command = new SqlCommand(query, connection, transaction); SqlParameter par = command.Parameters.Add(rowDataParameter, SqlDbType.NVarChar, -1); string rowData = JsonConvert.SerializeObject(rows); - this._logger.LogInformation(rowData); par.Value = rowData; return command; } From 940602cc488fc644d75f936585f54a8cbba7e67e Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 18 Oct 2022 11:29:10 -0700 Subject: [PATCH 50/77] simplify --- src/TriggerBinding/SqlTableChangeMonitor.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 01618a411..b1fb297e5 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -745,15 +745,10 @@ private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransa // These are the internal column values that we use. Note that we use SYS_CHANGE_VERSION because that's // the new version - the _az_func_ChangeVersion has the old version .Concat(new string[] { "SYS_CHANGE_VERSION bigint", "_az_func_AttemptCount int" }); - IList bracketedPrimaryKeys = this._primaryKeyColumns.Select(p => p.name.AsBracketQuotedString()).ToList(); + IEnumerable bracketedPrimaryKeys = this._primaryKeyColumns.Select(p => p.name.AsBracketQuotedString()); // Create the query that the merge statement will match the rows on - var primaryKeyMatchingQuery = new StringBuilder($"ExistingData.{bracketedPrimaryKeys[0]} = NewData.{bracketedPrimaryKeys[0]}"); - foreach (string primaryKey in bracketedPrimaryKeys.Skip(1)) - { - primaryKeyMatchingQuery.Append($" AND ExistingData.{primaryKey} = NewData.{primaryKey}"); - } - + string primaryKeyMatchingQuery = string.Join(" AND ", bracketedPrimaryKeys.Select(key => $"ExistingData.{key} = NewData.{key}")); const string acquireLeasesCte = "acquireLeasesCte"; const string rowDataParameter = "@rowData"; // Create the merge query that will either update the rows that already exist or insert a new one if it doesn't exist From 02a3c5ffcf1c2138dbf94e5fbf3e762c3cc91694 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 18 Oct 2022 12:13:19 -0700 Subject: [PATCH 51/77] fix compile --- src/TriggerBinding/SqlTableChangeMonitor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 136ea2345..a471f619e 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -777,7 +777,7 @@ LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCo /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildGetUnprocessedChangesCommand(SqlConnection connection) { - string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.AsBracketQuotedString()} = l.{col.AsBracketQuotedString()}")); + string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.name.AsBracketQuotedString()} = l.{col.name.AsBracketQuotedString()}")); string getUnprocessedChangesQuery = $@" DECLARE @last_sync_version bigint; From 1671e0825dbc2c9963008125a2cd7358675dda32 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 18 Oct 2022 13:14:49 -0700 Subject: [PATCH 52/77] Add const --- src/TriggerBinding/SqlTableChangeMonitor.cs | 24 ++++++++++----------- src/TriggerBinding/SqlTriggerConstants.cs | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index a471f619e..28353e4d3 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -645,7 +645,7 @@ private long RecomputeLastSyncVersion() var changeVersionSet = new SortedSet(); foreach (IReadOnlyDictionary row in this._rows) { - string changeVersion = row["SYS_CHANGE_VERSION"].ToString(); + string changeVersion = row[SqlTriggerConstants.SysChangeVersionColumnName].ToString(); changeVersionSet.Add(long.Parse(changeVersion, CultureInfo.InvariantCulture)); } @@ -753,17 +753,17 @@ private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransacti SELECT TOP {this._batchSize} {selectList}, - c.SYS_CHANGE_VERSION, c.SYS_CHANGE_OPERATION, + c.{SqlTriggerConstants.SysChangeVersionColumnName}, c.SYS_CHANGE_OPERATION, l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName}, l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName}, l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCondition} WHERE (l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NULL AND - (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.SYS_CHANGE_VERSION) OR + (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.{SqlTriggerConstants.SysChangeVersionColumnName}) OR l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}) - ORDER BY c.SYS_CHANGE_VERSION ASC; + ORDER BY c.{SqlTriggerConstants.SysChangeVersionColumnName} ASC; "; return new SqlCommand(getChangesQuery, connection, transaction); @@ -790,7 +790,7 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_ver LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} WHERE (l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NULL AND - (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.SYS_CHANGE_VERSION) OR + (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.{SqlTriggerConstants.SysChangeVersionColumnName}) OR l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}); "; @@ -813,7 +813,7 @@ private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransa .Select(c => $"{c.name.AsBracketQuotedString()} {c.type}") // These are the internal column values that we use. Note that we use SYS_CHANGE_VERSION because that's // the new version - the _az_func_ChangeVersion has the old version - .Concat(new string[] { "SYS_CHANGE_VERSION bigint", "_az_func_AttemptCount int" }); + .Concat(new string[] { $"{SqlTriggerConstants.SysChangeVersionColumnName} bigint", $"{SqlTriggerConstants.LeasesTableAttemptCountColumnName} int" }); IEnumerable bracketedPrimaryKeys = this._primaryKeyColumns.Select(p => p.name.AsBracketQuotedString()); // Create the query that the merge statement will match the rows on @@ -831,11 +831,11 @@ AS NewData {primaryKeyMatchingQuery} WHEN MATCHED THEN UPDATE SET - {SqlTriggerConstants.LeasesTableChangeVersionColumnName} = NewData.SYS_CHANGE_VERSION, + {SqlTriggerConstants.LeasesTableChangeVersionColumnName} = NewData.{SqlTriggerConstants.SysChangeVersionColumnName}, {SqlTriggerConstants.LeasesTableAttemptCountColumnName} = ExistingData.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} + 1, {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) WHEN NOT MATCHED THEN - INSERT VALUES ({string.Join(",", bracketedPrimaryKeys.Select(k => $"NewData.{k}"))}, NewData.SYS_CHANGE_VERSION, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()));"; + INSERT VALUES ({string.Join(",", bracketedPrimaryKeys.Select(k => $"NewData.{k}"))}, NewData.{SqlTriggerConstants.SysChangeVersionColumnName}, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()));"; var command = new SqlCommand(query, connection, transaction); SqlParameter par = command.Parameters.Add(rowDataParameter, SqlDbType.NVarChar, -1); @@ -875,7 +875,7 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa for (int rowIndex = 0; rowIndex < this._rows.Count; rowIndex++) { - string changeVersion = this._rows[rowIndex]["SYS_CHANGE_VERSION"].ToString(); + string changeVersion = this._rows[rowIndex][SqlTriggerConstants.SysChangeVersionColumnName].ToString(); releaseLeasesQuery.Append($@" SELECT @current_change_version = {SqlTriggerConstants.LeasesTableChangeVersionColumnName} @@ -916,13 +916,13 @@ private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, Sql DECLARE @unprocessed_changes bigint; SELECT @unprocessed_changes = COUNT(*) FROM ( - SELECT c.SYS_CHANGE_VERSION + SELECT c.{SqlTriggerConstants.SysChangeVersionColumnName} FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_sync_version) AS c LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} WHERE - c.SYS_CHANGE_VERSION <= {newLastSyncVersion} AND + c.{SqlTriggerConstants.SysChangeVersionColumnName} <= {newLastSyncVersion} AND ((l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR - l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} != c.SYS_CHANGE_VERSION OR + l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} != c.{SqlTriggerConstants.SysChangeVersionColumnName} OR l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NOT NULL) AND (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}))) AS Changes diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index 8353eb7c1..f79c617fa 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -14,7 +14,7 @@ internal static class SqlTriggerConstants public const string LeasesTableChangeVersionColumnName = "_az_func_ChangeVersion"; public const string LeasesTableAttemptCountColumnName = "_az_func_AttemptCount"; public const string LeasesTableLeaseExpirationTimeColumnName = "_az_func_LeaseExpirationTime"; - + public const string SysChangeVersionColumnName = "SYS_CHANGE_VERSION"; /// /// The column names that are used in internal state tables and so can't exist in the target table /// since that shares column names with the primary keys from each user table being monitored. From 5cd760a85dc29d18c0429c3fd8ca6a7b9aa7a2b7 Mon Sep 17 00:00:00 2001 From: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> Date: Sat, 22 Oct 2022 09:20:49 +0530 Subject: [PATCH 53/77] Add unit tests for runtime driven scaling (#414) --- src/TriggerBinding/SqlTriggerListener.cs | 17 +- .../SqlTriggerBindingProviderTests.cs} | 37 ++- .../TriggerBinding/SqlTriggerListenerTests.cs | 260 ++++++++++++++++++ 3 files changed, 298 insertions(+), 16 deletions(-) rename test/Unit/{SqlTriggerBindingTests.cs => TriggerBinding/SqlTriggerBindingProviderTests.cs} (64%) create mode 100644 test/Unit/TriggerBinding/SqlTriggerListenerTests.cs diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 954f133d4..359832f91 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -65,7 +65,9 @@ public SqlTriggerListener(string connectionString, string tableName, string user this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - this._scaleMonitorDescriptor = new ScaleMonitorDescriptor($"{userFunctionId}-SqlTrigger-{tableName}".ToLower(CultureInfo.InvariantCulture)); + // Do not convert the scale-monitor ID to lower-case string since SQL table names can be case-sensitive + // depending on the collation of the current database. + this._scaleMonitorDescriptor = new ScaleMonitorDescriptor($"{userFunctionId}-SqlTrigger-{tableName}"); } public void Cancel() @@ -541,8 +543,8 @@ private ScaleStatus GetScaleStatusCore(int workerCount, SqlTriggerMetrics[] metr // certain reliability. These samples roughly cover the timespan of past 40 seconds. const int minSamplesForScaling = 5; - // Please ensure the Readme file and other public documentation are also updated if this value ever needs to - // be changed. + // NOTE: please ensure the Readme file and other public documentation are also updated if this value ever + // needs to be changed. const int maxChangesPerWorker = 1000; var status = new ScaleStatus @@ -557,7 +559,10 @@ private ScaleStatus GetScaleStatusCore(int workerCount, SqlTriggerMetrics[] metr return status; } - string counts = string.Join(" ,", metrics.TakeLast(minSamplesForScaling).Select(metric => metric.UnprocessedChangeCount)); + // Consider only the most recent batch of samples in the rest of the method. + metrics = metrics.TakeLast(minSamplesForScaling).ToArray(); + + string counts = string.Join(", ", metrics.Select(metric => metric.UnprocessedChangeCount)); this._logger.LogInformation($"Unprocessed change counts: [{counts}], worker count: {workerCount}, maximum changes per worker: {maxChangesPerWorker}."); // Add worker if the count of unprocessed changes per worker exceeds the maximum limit. @@ -572,10 +577,10 @@ private ScaleStatus GetScaleStatusCore(int workerCount, SqlTriggerMetrics[] metr // Check if there is a continuous increase or decrease in count of unprocessed changes. bool isIncreasing = true; bool isDecreasing = true; - for (int index = metrics.Length - minSamplesForScaling; index < metrics.Length - 1; index++) + for (int index = 0; index < metrics.Length - 1; index++) { isIncreasing = isIncreasing && metrics[index].UnprocessedChangeCount < metrics[index + 1].UnprocessedChangeCount; - isDecreasing = isDecreasing && (metrics[index].UnprocessedChangeCount == 0 || metrics[index].UnprocessedChangeCount > metrics[index + 1].UnprocessedChangeCount); + isDecreasing = isDecreasing && (metrics[index + 1].UnprocessedChangeCount == 0 || metrics[index].UnprocessedChangeCount > metrics[index + 1].UnprocessedChangeCount); } if (isIncreasing) diff --git a/test/Unit/SqlTriggerBindingTests.cs b/test/Unit/TriggerBinding/SqlTriggerBindingProviderTests.cs similarity index 64% rename from test/Unit/SqlTriggerBindingTests.cs rename to test/Unit/TriggerBinding/SqlTriggerBindingProviderTests.cs index a874d34a8..1c0afd352 100644 --- a/test/Unit/SqlTriggerBindingTests.cs +++ b/test/Unit/TriggerBinding/SqlTriggerBindingProviderTests.cs @@ -15,18 +15,27 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Unit { - public class SqlTriggerBindingTests + public class SqlTriggerBindingProviderTests { + /// + /// Verifies that null trigger binding is returned if the trigger parameter in user function does not have + /// applied. + /// [Fact] - public async Task SqlTriggerBindingProvider_ReturnsNullBindingForParameterWithoutAttribute() + public async Task TryCreateAsync_TriggerParameterWithoutAttribute_ReturnsNullBinding() { Type parameterType = typeof(IReadOnlyList>); ITriggerBinding binding = await CreateTriggerBindingAsync(parameterType, nameof(UserFunctionWithoutAttribute)); Assert.Null(binding); } + /// + /// Verifies that is thrown if the applied on + /// the trigger parameter does not have property set. + /// attribute applied. + /// [Fact] - public async Task SqlTriggerBindingProvider_ThrowsForMissingConnectionString() + public async Task TryCreateAsync_MissingConnectionString_ThrowsException() { Type parameterType = typeof(IReadOnlyList>); Task testCode() { return CreateTriggerBindingAsync(parameterType, nameof(UserFunctionWithoutConnectionString)); } @@ -37,13 +46,17 @@ public async Task SqlTriggerBindingProvider_ThrowsForMissingConnectionString() exception.Message); } + /// + /// Verifies that is thrown if the is + /// applied on the trigger parameter of unsupported type. + /// [Theory] [InlineData(typeof(object))] [InlineData(typeof(SqlChange))] [InlineData(typeof(IEnumerable>))] [InlineData(typeof(IReadOnlyList))] [InlineData(typeof(IReadOnlyList>))] - public async Task SqlTriggerBindingProvider_ThrowsForInvalidTriggerParameterType(Type parameterType) + public async Task TryCreateAsync_InvalidTriggerParameterType_ThrowsException(Type parameterType) { Task testCode() { return CreateTriggerBindingAsync(parameterType, nameof(UserFunctionWithoutConnectionString)); } InvalidOperationException exception = await Assert.ThrowsAsync(testCode); @@ -53,23 +66,27 @@ public async Task SqlTriggerBindingProvider_ThrowsForInvalidTriggerParameterType exception.Message); } + /// + /// Verifies that is returned if the has all + /// required properties set and it is applied on the trigger parameter of supported type. + /// [Fact] - public async Task SqlTriggerBindingProvider_ReturnsBindingForValidTriggerParameterType() + public async Task TryCreateAsync_ValidTriggerParameterType_ReturnsTriggerBinding() { Type parameterType = typeof(IReadOnlyList>); ITriggerBinding binding = await CreateTriggerBindingAsync(parameterType, nameof(UserFunctionWithAttribute)); - Assert.NotNull(binding); + Assert.IsType>(binding); } private static async Task CreateTriggerBindingAsync(Type parameterType, string methodName) { var provider = new SqlTriggerBindingProvider( - Mock.Of(c => c["dummyConnectionStringSetting"] == "dummyConnectionString"), + Mock.Of(c => c["testConnectionStringSetting"] == "testConnectionString"), Mock.Of(), Mock.Of(f => f.CreateLogger(It.IsAny()) == Mock.Of())); // Possibly the simplest way to construct a ParameterInfo object. - ParameterInfo parameter = typeof(SqlTriggerBindingTests) + ParameterInfo parameter = typeof(SqlTriggerBindingProviderTests) .GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Static) .MakeGenericMethod(parameterType) .GetParameters()[0]; @@ -79,8 +96,8 @@ private static async Task CreateTriggerBindingAsync(Type parame private static void UserFunctionWithoutAttribute(T _) { } - private static void UserFunctionWithoutConnectionString([SqlTrigger("dummyTableName")] T _) { } + private static void UserFunctionWithoutConnectionString([SqlTrigger("testTableName")] T _) { } - private static void UserFunctionWithAttribute([SqlTrigger("dummyTableName", ConnectionStringSetting = "dummyConnectionStringSetting")] T _) { } + private static void UserFunctionWithAttribute([SqlTrigger("testTableName", ConnectionStringSetting = "testConnectionStringSetting")] T _) { } } } \ No newline at end of file diff --git a/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs b/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs new file mode 100644 index 000000000..28319363b --- /dev/null +++ b/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs @@ -0,0 +1,260 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Scale; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Internal; +using Moq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Unit +{ + public class SqlTriggerListenerTests + { + /// + /// Verifies that the scale monitor descriptor ID is set to expected value. + /// + [Theory] + [InlineData("testTableName", "testUserFunctionId", "testUserFunctionId-SqlTrigger-testTableName")] + [InlineData("тестТаблицаИмя", "тестПользовательФункцияИд", "тестПользовательФункцияИд-SqlTrigger-тестТаблицаИмя")] + public void ScaleMonitorDescriptor_ReturnsExpectedValue(string tableName, string userFunctionId, string expectedDescriptorId) + { + IScaleMonitor monitor = GetScaleMonitor(tableName, userFunctionId); + Assert.Equal(expectedDescriptorId, monitor.Descriptor.Id); + } + + /// + /// Verifies that no-scaling is requested if there are insufficient metrics available for making the scale + /// decision. + /// + [Theory] + [InlineData(null)] // metrics == null + [InlineData(new int[] { })] // metrics.Length == 0 + [InlineData(new int[] { 1000, 1000, 1000, 1000 })] // metrics.Length == 4. + public void ScaleMonitorGetScaleStatus_InsufficentMetrics_ReturnsNone(int[] unprocessedChangeCounts) + { + (IScaleMonitor monitor, List logMessages) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, 0); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(ScaleVote.None, scaleStatus.Vote); + Assert.Contains("Requesting no-scaling: Insufficient metrics for making scale decision for table: 'testTableName'.", logMessages); + } + + /// + /// Verifies that only the most recent samples are considered for making the scale decision. + /// + [Theory] + [InlineData(new int[] { 0, 0, 4, 3, 2, 0 }, 2, ScaleVote.None)] + [InlineData(new int[] { 0, 0, 4, 3, 2, 1, 0 }, 2, ScaleVote.ScaleIn)] + [InlineData(new int[] { 1000, 1000, 0, 1, 2, 1000 }, 1, ScaleVote.None)] + [InlineData(new int[] { 1000, 1000, 0, 1, 2, 3, 1000 }, 1, ScaleVote.ScaleOut)] + public void ScaleMonitorGetScaleStatus_ExcessMetrics_IgnoresExcessMetrics(int[] unprocessedChangeCounts, int workerCount, ScaleVote scaleVote) + { + (IScaleMonitor monitor, _) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, workerCount); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(scaleVote, scaleStatus.Vote); + } + + /// + /// Verifies that scale-out is requested if the latest count of unprocessed changes is above the combined limit + /// of all workers. + /// + [Theory] + [InlineData(new int[] { 0, 0, 0, 0, 1 }, 0)] + [InlineData(new int[] { 0, 0, 0, 0, 1001 }, 1)] + [InlineData(new int[] { 0, 0, 0, 0, 10001 }, 10)] + public void ScaleMonitorGetScaleStatus_LastCountAboveLimit_ReturnsScaleOut(int[] unprocessedChangeCounts, int workerCount) + { + (IScaleMonitor monitor, List logMessages) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, workerCount); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(ScaleVote.ScaleOut, scaleStatus.Vote); + Assert.Contains("Requesting scale-out: Found too many unprocessed changes for table: 'testTableName' relative to the number of workers.", logMessages); + } + + /// + /// Verifies that no-scaling is requested if the latest count of unprocessed changes is not above the combined + /// limit of all workers. + /// + [Theory] + [InlineData(new int[] { 0, 0, 0, 0, 0 }, 0)] + [InlineData(new int[] { 0, 0, 0, 0, 1000 }, 1)] + [InlineData(new int[] { 0, 0, 0, 0, 10000 }, 10)] + public void ScaleMonitorGetScaleStatus_LastCountBelowLimit_ReturnsNone(int[] unprocessedChangeCounts, int workerCount) + { + (IScaleMonitor monitor, List logMessages) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, workerCount); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(ScaleVote.None, scaleStatus.Vote); + Assert.Contains("Requesting no-scaling: Found the number of unprocessed changes for table: 'testTableName' to not require scaling.", logMessages); + } + + /// + /// Verifies that scale-out is requested if the count of unprocessed changes is strictly increasing and may + /// exceed the combined limit of all workers. Since the metric samples are separated by 10 seconds, the existing + /// implementation should only consider the last three samples in its calculation. + /// + [Theory] + [InlineData(new int[] { 0, 1, 500, 501, 751 }, 1)] + [InlineData(new int[] { 0, 1, 4999, 5001, 7500 }, 10)] + public void ScaleMonitorGetScaleStatus_CountIncreasingAboveLimit_ReturnsScaleOut(int[] unprocessedChangeCounts, int workerCount) + { + (IScaleMonitor monitor, List logMessages) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, workerCount); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(ScaleVote.ScaleOut, scaleStatus.Vote); + Assert.Contains("Requesting scale-out: Found the unprocessed changes for table: 'testTableName' to be continuously increasing and may exceed the maximum limit set for the workers.", logMessages); + } + + /// + /// Verifies that no-scaling is requested if the count of unprocessed changes is strictly increasing but it may + /// still stay below the combined limit of all workers. Since the metric samples are separated by 10 seconds, + /// the existing implementation should only consider the last three samples in its calculation. + /// + [Theory] + [InlineData(new int[] { 0, 1, 500, 501, 750 }, 1)] + [InlineData(new int[] { 0, 1, 5000, 5001, 7500 }, 10)] + public void ScaleMonitorGetScaleStatus_CountIncreasingBelowLimit_ReturnsNone(int[] unprocessedChangeCounts, int workerCount) + { + (IScaleMonitor monitor, List logMessages) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, workerCount); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(ScaleVote.None, scaleStatus.Vote); + Assert.Contains("Avoiding scale-out: Found the unprocessed changes for table: 'testTableName' to be increasing but they may not exceed the maximum limit set for the workers.", logMessages); + } + + /// + /// Verifies that scale-in is requested if the count of unprocessed changes is strictly decreasing (or zero) and + /// is also below the combined limit of workers after being reduced by one. + /// + [Theory] + [InlineData(new int[] { 0, 0, 0, 0, 0 }, 1)] + [InlineData(new int[] { 1, 0, 0, 0, 0 }, 1)] + [InlineData(new int[] { 5, 4, 3, 2, 0 }, 1)] + [InlineData(new int[] { 9005, 9004, 9003, 9002, 9000 }, 10)] + public void ScaleMonitorGetScaleStatus_CountDecreasingBelowLimit_ReturnsScaleIn(int[] unprocessedChangeCounts, int workerCount) + { + (IScaleMonitor monitor, List logMessages) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, workerCount); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(ScaleVote.ScaleIn, scaleStatus.Vote); + Assert.Contains("Requesting scale-in: Found table: 'testTableName' to be either idle or the unprocessed changes to be continuously decreasing.", logMessages); + } + + /// + /// Verifies that scale-in is requested if the count of unprocessed changes is strictly decreasing (or zero) but + /// it is still above the combined limit of workers after being reduced by one. + /// + [Theory] + [InlineData(new int[] { 5, 4, 3, 2, 1 }, 1)] + [InlineData(new int[] { 9005, 9004, 9003, 9002, 9001 }, 10)] + public void ScaleMonitorGetScaleStatus_CountDecreasingAboveLimit_ReturnsNone(int[] unprocessedChangeCounts, int workerCount) + { + (IScaleMonitor monitor, List logMessages) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, workerCount); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(ScaleVote.None, scaleStatus.Vote); + Assert.Contains("Avoiding scale-in: Found the unprocessed changes for table: 'testTableName' to be decreasing but they are high enough to require all existing workers for processing.", logMessages); + } + + /// + /// Verifies that no-scaling is requested if the count of unprocessed changes is neither strictly increasing and + /// nor strictly decreasing. + /// + [Theory] + [InlineData(new int[] { 0, 0, 1, 2, 3 }, 1)] + [InlineData(new int[] { 1, 1, 0, 0, 0 }, 10)] + public void ScaleMonitorGetScaleStatus_CountNotIncreasingOrDecreasing_ReturnsNone(int[] unprocessedChangeCounts, int workerCount) + { + (IScaleMonitor monitor, List logMessages) = GetScaleMonitor(); + ScaleStatusContext context = GetScaleStatusContext(unprocessedChangeCounts, workerCount); + + ScaleStatus scaleStatus = monitor.GetScaleStatus(context); + + Assert.Equal(ScaleVote.None, scaleStatus.Vote); + + // Ensure that no-scaling was not requested because of other conditions. + Assert.DoesNotContain("Avoiding scale-out: Found the unprocessed changes for table: 'testTableName' to be increasing but they may not exceed the maximum limit set for the workers.", logMessages); + Assert.DoesNotContain("Avoiding scale-in: Found the unprocessed changes for table: 'testTableName' to be decreasing but they are high enough to require all existing workers for processing.", logMessages); + Assert.Contains("Requesting no-scaling: Found the number of unprocessed changes for table: 'testTableName' to not require scaling.", logMessages); + } + + private static IScaleMonitor GetScaleMonitor(string tableName, string userFunctionId) + { + return new SqlTriggerListener( + "testConnectionString", + tableName, + userFunctionId, + Mock.Of(), + Mock.Of(), + Mock.Of()); + } + + private static (IScaleMonitor monitor, List logMessages) GetScaleMonitor() + { + // Since multiple threads are not involved when computing the scale-status, it should be okay to not use + // a thread-safe collection for storing the log messages. + var logMessages = new List(); + var mockLogger = new Mock(); + + // Both LogInformation() and LogDebug() are extension methods. Since the extension methods are static, they + // cannot be mocked. Hence, we need to setup callback on an inner class method that gets eventually called + // by these methods in order to extract the log message. + mockLogger + .Setup(logger => logger.Log(It.IsAny(), 0, It.IsAny(), null, It.IsAny>())) + .Callback((LogLevel logLevel, EventId eventId, object state, Exception exception, Func formatter) => + { + logMessages.Add(state.ToString()); + }); + + IScaleMonitor monitor = new SqlTriggerListener( + "testConnectionString", + "testTableName", + "testUserFunctionId", + Mock.Of(), + mockLogger.Object, + Mock.Of()); + + return (monitor, logMessages); + } + + private static ScaleStatusContext GetScaleStatusContext(int[] unprocessedChangeCounts, int workerCount) + { + DateTime now = DateTime.UtcNow; + + // Returns metric samples separated by 10 seconds. The time-difference is essential for testing the + // scale-out logic. + return new ScaleStatusContext + { + Metrics = unprocessedChangeCounts?.Select((count, index) => new SqlTriggerMetrics + { + UnprocessedChangeCount = count, + Timestamp = now + TimeSpan.FromSeconds(10 * index), + }), + WorkerCount = workerCount, + }; + } + } +} \ No newline at end of file From 55639aa810889922e73c23bd5142939864dd48b6 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 28 Oct 2022 10:10:20 -0700 Subject: [PATCH 54/77] Merge branch 'main' into triggerbindings --- Directory.Build.props | 2 +- Directory.Packages.props | 18 +- README.md | 1 + WebJobs.Extensions.Sql.sln | 9 + ...zure.Functions.Worker.Extension.Sql.csproj | 21 + .../src/Properties/AssemblyInfo.cs | 6 + Worker.Extension.Sql/src/SqlInputAttribute.cs | 53 ++ .../src/SqlOutputAttribute.cs | 41 ++ Worker.Extension.Sql/src/packages.lock.json | 401 ++++++++++++ builds/azure-pipelines/build-release.yml | 6 +- java-library/pom.xml | 8 - ....WebJobs.Extensions.Sql.Performance.csproj | 1 - performance/packages.lock.json | 594 +++++++++--------- ...zure.WebJobs.Extensions.Sql.Samples.csproj | 2 +- .../AddProductWithIdentityColumnIncluded.cs | 2 +- samples/samples-csharp/global.json | 6 - samples/samples-csharp/packages.lock.json | 582 +++++++++-------- src/SqlAsyncCollector.cs | 3 +- src/SqlAsyncEnumerable.cs | 7 +- src/SqlAttribute.cs | 4 +- src/SqlBindingConstants.cs | 10 + src/SqlConverters.cs | 8 +- src/packages.lock.json | 353 ++++++----- .../Common/SupportedLanguagesTestAttribute.cs | 2 +- test/Common/TestData.cs | 2 +- test/Common/TestUtils.cs | 2 +- test/GlobalSuppressions.cs | 4 +- test/Integration/IntegrationTestBase.cs | 3 +- .../SqlInputBindingIntegrationTests.cs | 36 +- .../GetProductColumnTypesSerialization.cs | 12 +- ...olumnTypesSerializationAsyncEnumerable.cs} | 23 +- test/Unit/SqlInputBindingTests.cs | 18 +- test/Unit/SqlOutputBindingTests.cs | 4 +- test/packages.lock.json | 588 ++++++++--------- 34 files changed, 1755 insertions(+), 1077 deletions(-) create mode 100644 Worker.Extension.Sql/src/Microsoft.Azure.Functions.Worker.Extension.Sql.csproj create mode 100644 Worker.Extension.Sql/src/Properties/AssemblyInfo.cs create mode 100644 Worker.Extension.Sql/src/SqlInputAttribute.cs create mode 100644 Worker.Extension.Sql/src/SqlOutputAttribute.cs create mode 100644 Worker.Extension.Sql/src/packages.lock.json delete mode 100644 samples/samples-csharp/global.json create mode 100644 src/SqlBindingConstants.cs rename test/Integration/test-csharp/{GetProductColumnTypesSerializationDifferentCulture.cs => GetProductColumnTypesSerializationAsyncEnumerable.cs} (63%) diff --git a/Directory.Build.props b/Directory.Build.props index 50110af53..57582ee80 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,6 +1,6 @@ - netcoreapp3.1 + net6 true true true diff --git a/Directory.Packages.props b/Directory.Packages.props index 1211997da..3945520c6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,20 +1,22 @@ - - - - + + + + + - + + - - - + + + \ No newline at end of file diff --git a/README.md b/README.md index c81ab170f..e2cbd75c1 100644 --- a/README.md +++ b/README.md @@ -948,6 +948,7 @@ public static void Run( ## Known Issues - Output bindings against tables with columns of data types `NTEXT`, `TEXT`, or `IMAGE` are not supported and data upserts will fail. These types [will be removed](https://docs.microsoft.com/sql/t-sql/data-types/ntext-text-and-image-transact-sql) in a future version of SQL Server and are not compatible with the `OPENJSON` function used by this Azure Functions binding. +- Input bindings against tables with columns of data types 'DATETIME', 'DATETIME2', or 'SMALLDATETIME' will assume that the values are in UTC format. - Trigger bindings will exhibit undefined behavior if the SQL table schema gets modified while the user application is running, for example, if a column is added, renamed or deleted or if the primary key is modified or deleted. In such cases, restarting the application should help resolve any errors. diff --git a/WebJobs.Extensions.Sql.sln b/WebJobs.Extensions.Sql.sln index 7600df817..ef23ebde5 100644 --- a/WebJobs.Extensions.Sql.sln +++ b/WebJobs.Extensions.Sql.sln @@ -24,6 +24,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.WebJobs.Ext EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "performance", "performance", "{F0F3562F-9176-4461-98E4-13D38D3DD056}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Worker.Extension.Sql", "Worker.Extension.Sql", "{605E19C0-3A77-477F-928E-85B8972B734D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Azure.Functions.Worker.Extension.Sql", "Worker.Extension.Sql\src\Microsoft.Azure.Functions.Worker.Extension.Sql.csproj", "{84D97605-F1BF-4083-9C93-2B68A9FBB00F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -46,6 +50,10 @@ Global {1A5148B7-F877-4813-852C-F94D6EF795E8}.Debug|Any CPU.Build.0 = Debug|Any CPU {1A5148B7-F877-4813-852C-F94D6EF795E8}.Release|Any CPU.ActiveCfg = Release|Any CPU {1A5148B7-F877-4813-852C-F94D6EF795E8}.Release|Any CPU.Build.0 = Release|Any CPU + {84D97605-F1BF-4083-9C93-2B68A9FBB00F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84D97605-F1BF-4083-9C93-2B68A9FBB00F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84D97605-F1BF-4083-9C93-2B68A9FBB00F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84D97605-F1BF-4083-9C93-2B68A9FBB00F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -55,6 +63,7 @@ Global {4453B407-2CA3-4011-BDE5-247FCE5C1974} = {21CDC01C-247B-46B0-A8F5-9D35686C628B} {A5B55530-71C8-4A9A-92AD-1D33A5E80FC1} = {F7E99EB5-47D3-4B50-A6AA-D8D5508A121A} {1A5148B7-F877-4813-852C-F94D6EF795E8} = {F0F3562F-9176-4461-98E4-13D38D3DD056} + {84D97605-F1BF-4083-9C93-2B68A9FBB00F} = {605E19C0-3A77-477F-928E-85B8972B734D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {49902AA5-150F-4567-B562-4AA8549B2CF4} diff --git a/Worker.Extension.Sql/src/Microsoft.Azure.Functions.Worker.Extension.Sql.csproj b/Worker.Extension.Sql/src/Microsoft.Azure.Functions.Worker.Extension.Sql.csproj new file mode 100644 index 000000000..6e5d11591 --- /dev/null +++ b/Worker.Extension.Sql/src/Microsoft.Azure.Functions.Worker.Extension.Sql.csproj @@ -0,0 +1,21 @@ + + + + Microsoft.Azure.Functions.Worker.Extension.Sql + Microsoft.Azure.Functions.Worker.Extension.Sql + Sql extension for .NET isolated Azure Functions + + + 99.99.99 + + + false + + + + + + + + + \ No newline at end of file diff --git a/Worker.Extension.Sql/src/Properties/AssemblyInfo.cs b/Worker.Extension.Sql/src/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..95312e274 --- /dev/null +++ b/Worker.Extension.Sql/src/Properties/AssemblyInfo.cs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + +[assembly: ExtensionInformation("Microsoft.Azure.WebJobs.Extensions.Sql", "0.*-preview")] \ No newline at end of file diff --git a/Worker.Extension.Sql/src/SqlInputAttribute.cs b/Worker.Extension.Sql/src/SqlInputAttribute.cs new file mode 100644 index 000000000..dec0d7df8 --- /dev/null +++ b/Worker.Extension.Sql/src/SqlInputAttribute.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + +namespace Microsoft.Azure.Functions.Worker.Extension.Sql +{ + public sealed class SqlInputAttribute : InputBindingAttribute + { + /// + /// Creates an instance of the , specifying the Sql attributes + /// the function supports. + /// + /// The text of the command. + public SqlInputAttribute(string commandText) + { + this.CommandText = commandText; + } + + /// + /// The name of the app setting where the SQL connection string is stored + /// (see https://docs.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection). + /// The attributes specified in the connection string are listed here + /// https://docs.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring + /// For example, to create a connection to the "TestDB" located at the URL "test.database.windows.net" using a User ID and password, + /// create a ConnectionStringSetting with a name like SqlServerAuthentication. The value of the SqlServerAuthentication app setting + /// would look like "Data Source=test.database.windows.net;Database=TestDB;User ID={userid};Password={password}". + /// + public string ConnectionStringSetting { get; set; } + + /// + /// Either a SQL query or stored procedure that will be run in the database referred to in the ConnectionString. + /// + public string CommandText { get; set; } + + /// + /// Specifies whether refers to a stored procedure or SQL query string. + /// Use for the former, for the latter + /// + public System.Data.CommandType CommandType { get; set; } = System.Data.CommandType.Text; + + /// + /// Specifies the parameters that will be used to execute the SQL query or stored procedure specified in . + /// Must follow the format "@param1=param1,@param2=param2". For example, if your SQL query looks like + /// "select * from Products where cost = @Cost and name = @Name", then Parameters must have the form "@Cost=100,@Name={Name}" + /// If the value of a parameter should be null, use "null", as in @param1=null,@param2=param2". + /// If the value of a parameter should be an empty string, do not add anything after the equals sign and before the comma, + /// as in "@param1=,@param2=param2" + /// Note that neither the parameter name nor the parameter value can have ',' or '=' + /// + public string Parameters { get; set; } + } +} diff --git a/Worker.Extension.Sql/src/SqlOutputAttribute.cs b/Worker.Extension.Sql/src/SqlOutputAttribute.cs new file mode 100644 index 000000000..55c131f51 --- /dev/null +++ b/Worker.Extension.Sql/src/SqlOutputAttribute.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.Functions.Worker.Extensions.Abstractions; + +namespace Microsoft.Azure.Functions.Worker.Extension.Sql +{ + public class SqlOutputAttribute : OutputBindingAttribute + { + /// + /// Creates an instance of the , specifying the Sql attributes + /// the function supports. + /// + /// The text of the command. + public SqlOutputAttribute(string commandText) + { + this.CommandText = commandText; + } + + /// + /// The name of the app setting where the SQL connection string is stored + /// (see https://docs.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection). + /// The attributes specified in the connection string are listed here + /// https://docs.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring + /// For example, to create a connection to the "TestDB" located at the URL "test.database.windows.net" using a User ID and password, + /// create a ConnectionStringSetting with a name like SqlServerAuthentication. The value of the SqlServerAuthentication app setting + /// would look like "Data Source=test.database.windows.net;Database=TestDB;User ID={userid};Password={password}". + /// + public string ConnectionStringSetting { get; set; } + + /// + /// The table name to upsert the values to. + /// + public string CommandText { get; set; } + + /// + /// Specifies as Text. + /// + public System.Data.CommandType CommandType { get; } = System.Data.CommandType.Text; + } +} diff --git a/Worker.Extension.Sql/src/packages.lock.json b/Worker.Extension.Sql/src/packages.lock.json new file mode 100644 index 000000000..de37638c4 --- /dev/null +++ b/Worker.Extension.Sql/src/packages.lock.json @@ -0,0 +1,401 @@ +{ + "version": 2, + "dependencies": { + "net6.0": { + "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": { + "type": "Direct", + "requested": "[1.1.0, )", + "resolved": "1.1.0", + "contentHash": "kAs9BTuzdOvyuN2m5CYyQzyvzXKJ6hhIOgcwm0W8Q+Fwj91a1eBmRSi9pVzpM4V3skNt/+pkPD3wxFD4nEw0bg==" + }, + "Microsoft.Data.SqlClient": { + "type": "Direct", + "requested": "[5.0.1, )", + "resolved": "5.0.1", + "contentHash": "uu8dfrsx081cSbEevWuZAvqdmANDGJkbLBL2G3j0LAZxX1Oy8RCVAaC4Lcuak6jNicWP6CWvHqBTIEmQNSxQlw==", + "dependencies": { + "Azure.Identity": "1.6.0", + "Microsoft.Data.SqlClient.SNI.runtime": "5.0.1", + "Microsoft.Identity.Client": "4.45.0", + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.21.0", + "Microsoft.SqlServer.Server": "1.0.0", + "Microsoft.Win32.Registry": "5.0.0", + "System.Buffers": "4.5.1", + "System.Configuration.ConfigurationManager": "5.0.0", + "System.Diagnostics.DiagnosticSource": "5.0.0", + "System.IO": "4.3.0", + "System.Resources.ResourceManager": "4.3.0", + "System.Runtime.Caching": "5.0.0", + "System.Security.Cryptography.Cng": "5.0.0", + "System.Security.Principal.Windows": "5.0.0", + "System.Text.Encoding.CodePages": "5.0.0", + "System.Text.Encodings.Web": "4.7.2" + } + }, + "System.Drawing.Common": { + "type": "Direct", + "requested": "[5.0.3, )", + "resolved": "5.0.3", + "contentHash": "rEQZuslijqdsO0pkJn7LtGBaMc//YVA8de0meGihkg9oLPaN+w+/Pb5d71lgp0YjPoKgBKNMvdq0IPnoW4PEng==", + "dependencies": { + "Microsoft.Win32.SystemEvents": "5.0.0" + } + }, + "Azure.Core": { + "type": "Transitive", + "resolved": "1.24.0", + "contentHash": "+/qI1j2oU1S4/nvxb2k/wDsol00iGf1AyJX5g3epV7eOpQEP/2xcgh/cxgKMeFgn3U2fmgSiBnQZdkV+l5y0Uw==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "1.1.1", + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory.Data": "1.0.2", + "System.Numerics.Vectors": "4.5.0", + "System.Text.Encodings.Web": "4.7.2", + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, + "Azure.Identity": { + "type": "Transitive", + "resolved": "1.6.0", + "contentHash": "EycyMsb6rD2PK9P0SyibFfEhvWWttdrYhyPF4f41uzdB/44yQlV+2Wehxyg489Rj6gbPvSPgbKq0xsHJBhipZA==", + "dependencies": { + "Azure.Core": "1.24.0", + "Microsoft.Identity.Client": "4.39.0", + "Microsoft.Identity.Client.Extensions.Msal": "2.19.3", + "System.Memory": "4.5.4", + "System.Security.Cryptography.ProtectedData": "4.7.0", + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "1.1.1", + "contentHash": "yuvf07qFWFqtK3P/MRkEKLhn5r2UbSpVueRziSqj0yJQIKFwG1pq9mOayK3zE5qZCTs0CbrwL9M6R8VwqyGy2w==" + }, + "Microsoft.CSharp": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "kaj6Wb4qoMuH3HySFJhxwQfe8R/sJsNJnANrvv8WdFPMoNbKY5htfNscv+LHCu5ipz+49m2e+WQXpLXr9XYemQ==" + }, + "Microsoft.Data.SqlClient.SNI.runtime": { + "type": "Transitive", + "resolved": "5.0.1", + "contentHash": "y0X5MxiNdbITJYoafJ2ruaX6hqO0twpCGR/ipiDOe85JKLU8WL4TuAQfDe5qtt3bND5Je26HnrarLSAMMnVTNg==" + }, + "Microsoft.Identity.Client": { + "type": "Transitive", + "resolved": "4.45.0", + "contentHash": "ircobISCLWbtE5eEoLKU+ldfZ8O41vg4lcy38KRj/znH17jvBiAl8oxcyNp89CsuqE3onxIpn21Ca7riyDDrRw==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.18.0" + } + }, + "Microsoft.Identity.Client.Extensions.Msal": { + "type": "Transitive", + "resolved": "2.19.3", + "contentHash": "zVVZjn8aW7W79rC1crioDgdOwaFTQorsSO6RgVlDDjc7MvbEGz071wSNrjVhzR0CdQn6Sefx7Abf1o7vasmrLg==", + "dependencies": { + "Microsoft.Identity.Client": "4.38.0", + "System.Security.Cryptography.ProtectedData": "4.5.0" + } + }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "XeE6LQtD719Qs2IG7HDi1TSw9LIkDbJ33xFiOBoHbApVw/8GpIBCbW+t7RwOjErUDyXZvjhZliwRkkLb8Z1uzg==" + }, + "Microsoft.IdentityModel.JsonWebTokens": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "d3h1/BaMeylKTkdP6XwRCxuOoDJZ44V9xaXr6gl5QxmpnZGdoK3bySo3OQN8ehRLJHShb94ElLUvoXyglQtgAw==", + "dependencies": { + "Microsoft.IdentityModel.Tokens": "6.21.0" + } + }, + "Microsoft.IdentityModel.Logging": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "tuEhHIQwvBEhMf8I50hy8FHmRSUkffDFP5EdLsSDV4qRcl2wvOPkQxYqEzWkh+ytW6sbdJGEXElGhmhDfAxAKg==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.21.0" + } + }, + "Microsoft.IdentityModel.Protocols": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "0FqY5cTLQKtHrClzHEI+QxJl8OBT2vUiEQQB7UKk832JDiJJmetzYZ3AdSrPjN/3l3nkhByeWzXnhrX0JbifKg==", + "dependencies": { + "Microsoft.IdentityModel.Logging": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" + } + }, + "Microsoft.IdentityModel.Protocols.OpenIdConnect": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "vtSKL7n6EnAsLyxmiviusm6LKrblT2ndnNqN6rvVq6iIHAnPCK9E2DkDx6h1Jrpy1cvbp40r0cnTg23nhEAGTA==", + "dependencies": { + "Microsoft.IdentityModel.Protocols": "6.21.0", + "System.IdentityModel.Tokens.Jwt": "6.21.0" + } + }, + "Microsoft.IdentityModel.Tokens": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "AAEHZvZyb597a+QJSmtxH3n2P1nIJGpZ4Q89GTenknRx6T6zyfzf592yW/jA5e8EHN4tNMjjXHQaYWEq5+L05w==", + "dependencies": { + "Microsoft.CSharp": "4.5.0", + "Microsoft.IdentityModel.Logging": "6.21.0", + "System.Security.Cryptography.Cng": "4.5.0" + } + }, + "Microsoft.NETCore.Platforms": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "VyPlqzH2wavqquTcYpkIIAQ6WdenuKoFN0BdYBbCWsclXacSOHNQn66Gt4z5NBqEYW0FAPm5rlvki9ZiCij5xQ==" + }, + "Microsoft.NETCore.Targets": { + "type": "Transitive", + "resolved": "1.1.0", + "contentHash": "aOZA3BWfz9RXjpzt0sRJJMjAscAUm3Hoa4UWAfceV9UTYxgwZ1lZt5nO2myFf+/jetYQo4uTP7zS8sJY67BBxg==" + }, + "Microsoft.SqlServer.Server": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "N4KeF3cpcm1PUHym1RmakkzfkEv3GRMyofVv40uXsQhCQeglr2OHNcUk2WOG51AKpGO8ynGpo9M/kFXSzghwug==" + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", + "dependencies": { + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "Microsoft.Win32.SystemEvents": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "Bh6blKG8VAKvXiLe2L+sEsn62nc1Ij34MrNxepD2OCrS5cpCwQa9MeLyhVQPQ/R4Wlzwuy6wMK8hLb11QPDRsQ==", + "dependencies": { + "Microsoft.NETCore.Platforms": "5.0.0" + } + }, + "System.Buffers": { + "type": "Transitive", + "resolved": "4.5.1", + "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==" + }, + "System.Configuration.ConfigurationManager": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "aM7cbfEfVNlEEOj3DsZP+2g9NRwbkyiAv2isQEzw7pnkDg9ekCU2m1cdJLM02Uq691OaCS91tooaxcEn8d0q5w==", + "dependencies": { + "System.Security.Cryptography.ProtectedData": "5.0.0", + "System.Security.Permissions": "5.0.0" + } + }, + "System.Diagnostics.DiagnosticSource": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==" + }, + "System.Formats.Asn1": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "MTvUIktmemNB+El0Fgw9egyqT9AYSIk6DTJeoDSpc3GIHxHCMo8COqkWT1mptX5tZ1SlQ6HJZ0OsSvMth1c12w==" + }, + "System.Globalization": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "kYdVd2f2PAdFGblzFswE4hkNANJBKRmsfa2X5LG2AcWE1c7/4t0pYae1L8vfZ5xvE2nK/R9JprtToA61OSHWIg==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0", + "System.Runtime": "4.3.0" + } + }, + "System.IdentityModel.Tokens.Jwt": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "JRD8AuypBE+2zYxT3dMJomQVsPYsCqlyZhWel3J1d5nzQokSRyTueF+Q4ID3Jcu6zSZKuzOdJ1MLTkbQsDqcvQ==", + "dependencies": { + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" + } + }, + "System.IO": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "3qjaHvxQPDpSOYICjUoTsmoq5u6QJAFRUITgeT/4gqkF1bajbSmb1kwSxEA8AHlofqgcKJcM8udgieRNhaJ5Cg==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0", + "System.Runtime": "4.3.0", + "System.Text.Encoding": "4.3.0", + "System.Threading.Tasks": "4.3.0" + } + }, + "System.Memory": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "1MbJTHS1lZ4bS4FmsJjnuGJOu88ZzTT2rLvrhW7Ygic+pC0NWA+3hgAen0HRdsocuQXCkUTdFn9yHJJhsijDXw==" + }, + "System.Memory.Data": { + "type": "Transitive", + "resolved": "1.0.2", + "contentHash": "JGkzeqgBsiZwKJZ1IxPNsDFZDhUvuEdX8L8BDC8N3KOj+6zMcNU28CNN59TpZE/VJYy9cP+5M+sbxtWJx3/xtw==", + "dependencies": { + "System.Text.Encodings.Web": "4.7.2", + "System.Text.Json": "4.6.0" + } + }, + "System.Numerics.Vectors": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "QQTlPTl06J/iiDbJCiepZ4H//BVraReU4O4EoRw1U02H5TLUIT7xn3GnDp9AXPSlJUDyFs4uWjWafNX6WrAojQ==" + }, + "System.Reflection": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "KMiAFoW7MfJGa9nDFNcfu+FpEdiHpWgTcS2HdMpDvt9saK3y/G4GwprPyzqjFH9NTaGPQeWNHU+iDlDILj96aQ==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0", + "System.IO": "4.3.0", + "System.Reflection.Primitives": "4.3.0", + "System.Runtime": "4.3.0" + } + }, + "System.Reflection.Primitives": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "5RXItQz5As4xN2/YUDxdpsEkMhvw3e6aNveFXUn4Hl/udNTCNhnKp8lT9fnc3MhvGKh1baak5CovpuQUXHAlIA==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0", + "System.Runtime": "4.3.0" + } + }, + "System.Resources.ResourceManager": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "/zrcPkkWdZmI4F92gL/TPumP98AVDu/Wxr3CSJGQQ+XN6wbRZcyfSKVoPo17ilb3iOr0cCRqJInGwNMolqhS8A==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0", + "System.Globalization": "4.3.0", + "System.Reflection": "4.3.0", + "System.Runtime": "4.3.0" + } + }, + "System.Runtime": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "JufQi0vPQ0xGnAczR13AUFglDyVYt4Kqnz1AZaiKZ5+GICq0/1MH/mO/eAJHt/mHW1zjKBJd7kV26SrxddAhiw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0" + } + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "System.Security.Cryptography.Cng": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "jIMXsKn94T9JY7PvPq/tMfqa6GAaHpElRDpmG+SuL+D3+sTw2M8VhnibKnN8Tq+4JqbPJ/f+BwtLeDMEnzAvRg==", + "dependencies": { + "System.Formats.Asn1": "5.0.0" + } + }, + "System.Security.Cryptography.ProtectedData": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "HGxMSAFAPLNoxBvSfW08vHde0F9uh7BjASwu6JF9JnXuEPhCY3YUqURn0+bQV/4UWeaqymmrHWV+Aw9riQCtCA==" + }, + "System.Security.Permissions": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "uE8juAhEkp7KDBCdjDIE3H9R1HJuEHqeqX8nLX9gmYKWwsqk3T5qZlPx8qle5DPKimC/Fy3AFTdV7HamgCh9qQ==", + "dependencies": { + "System.Security.AccessControl": "5.0.0", + "System.Windows.Extensions": "5.0.0" + } + }, + "System.Security.Principal.Windows": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" + }, + "System.Text.Encoding": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "BiIg+KWaSDOITze6jGQynxg64naAPtqGHBwDrLaCtixsa5bKiR8dpPOHA7ge3C0JJQizJE+sfkz1wV+BAKAYZw==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0", + "System.Runtime": "4.3.0" + } + }, + "System.Text.Encoding.CodePages": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "NyscU59xX6Uo91qvhOs2Ccho3AR2TnZPomo1Z0K6YpyztBPM/A5VbkzOO19sy3A3i1TtEnTxA7bCe3Us+r5MWg==", + "dependencies": { + "Microsoft.NETCore.Platforms": "5.0.0" + } + }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "4.7.2", + "contentHash": "iTUgB/WtrZ1sWZs84F2hwyQhiRH6QNjQv2DkwrH+WP6RoFga2Q1m3f9/Q7FG8cck8AdHitQkmkXSY8qylcDmuA==" + }, + "System.Text.Json": { + "type": "Transitive", + "resolved": "4.7.2", + "contentHash": "TcMd95wcrubm9nHvJEQs70rC0H/8omiSGGpU4FQ/ZA1URIqD4pjmFJh2Mfv1yH1eHgJDWTi2hMDXwTET+zOOyg==" + }, + "System.Threading.Tasks": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "LbSxKEdOUhVe8BezB/9uOGGppt+nZf6e1VFyw6v3DN6lqitm0OSn2uXMOdtP0M3W4iMcqcivm2J6UgqiwwnXiA==", + "dependencies": { + "Microsoft.NETCore.Platforms": "1.1.0", + "Microsoft.NETCore.Targets": "1.1.0", + "System.Runtime": "4.3.0" + } + }, + "System.Threading.Tasks.Extensions": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" + }, + "System.Windows.Extensions": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "c1ho9WU9ZxMZawML+ssPKZfdnrg/OjR3pe0m9v8230z3acqphwvPJqzAkH54xRYm5ntZHGG1EPP3sux9H3qSPg==", + "dependencies": { + "System.Drawing.Common": "5.0.0" + } + }, + "System.Runtime.Caching": { + "type": "CentralTransitive", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "30D6MkO8WF9jVGWZIP0hmCN8l9BTY4LCsAzLIe4xFSXzs+AjDotR7DpSmj27pFskDURzUvqYYY0ikModgBTxWw==", + "dependencies": { + "System.Configuration.ConfigurationManager": "5.0.0" + } + } + } + } +} \ No newline at end of file diff --git a/builds/azure-pipelines/build-release.yml b/builds/azure-pipelines/build-release.yml index 63697d36d..d866954fc 100644 --- a/builds/azure-pipelines/build-release.yml +++ b/builds/azure-pipelines/build-release.yml @@ -13,10 +13,10 @@ schedules: variables: solution: '**/*.sln' configuration: 'Release' - versionMajor: 0 # TODO When this is bumped have versionPatch counter use versionMajorMinor instead https://github.com/Azure/azure-functions-sql-extension/issues/173 - versionMinor: 1 + versionMajor: 1 + versionMinor: 0 versionMajorMinor: '$(versionMajor).$(versionMinor)' # This variable is only used for the counter so we reset properly when either major or minor is bumped - versionPatch: $[counter(variables['versionMinor'], 0)] # This will reset when we bump minor version + versionPatch: $[counter(variables['versionMajorMinor'], 0)] # This will reset when we bump minor version binariesVersion: '$(versionMajor).$(versionMinor).$(versionPatch)' nugetVersion: '$(binariesVersion)-preview' diff --git a/java-library/pom.xml b/java-library/pom.xml index 0392c154a..08c106bec 100644 --- a/java-library/pom.xml +++ b/java-library/pom.xml @@ -40,14 +40,6 @@ HEAD - - - LucyZhang - Lucy Zhang - luczhan@microsoft.com - - - ossrh diff --git a/performance/Microsoft.Azure.WebJobs.Extensions.Sql.Performance.csproj b/performance/Microsoft.Azure.WebJobs.Extensions.Sql.Performance.csproj index 19f8e9efe..3c779205a 100644 --- a/performance/Microsoft.Azure.WebJobs.Extensions.Sql.Performance.csproj +++ b/performance/Microsoft.Azure.WebJobs.Extensions.Sql.Performance.csproj @@ -2,7 +2,6 @@ Exe - netcoreapp3.1 diff --git a/performance/packages.lock.json b/performance/packages.lock.json index f72a67a28..cf3686792 100644 --- a/performance/packages.lock.json +++ b/performance/packages.lock.json @@ -1,7 +1,7 @@ { "version": 2, "dependencies": { - ".NETCoreApp,Version=v3.1": { + "net6.0": { "BenchmarkDotNet": { "type": "Direct", "requested": "[0.13.1, )", @@ -27,32 +27,30 @@ }, "Azure.Core": { "type": "Transitive", - "resolved": "1.19.0", - "contentHash": "lcDjG635DPE4fU5tqSueVMmzrx0QrIfPuY0+y6evHN5GanQ0GB+/4nuMHMmoNPwEow6OUPkJu4cZQxfHJQXPdA==", + "resolved": "1.24.0", + "contentHash": "+/qI1j2oU1S4/nvxb2k/wDsol00iGf1AyJX5g3epV7eOpQEP/2xcgh/cxgKMeFgn3U2fmgSiBnQZdkV+l5y0Uw==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "1.0.0", - "System.Buffers": "4.5.1", + "Microsoft.Bcl.AsyncInterfaces": "1.1.1", "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4", "System.Memory.Data": "1.0.2", "System.Numerics.Vectors": "4.5.0", "System.Text.Encodings.Web": "4.7.2", - "System.Text.Json": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Azure.Identity": { "type": "Transitive", - "resolved": "1.4.0", - "contentHash": "vvjdoDQb9WQyLkD1Uo5KFbwlW7xIsDMihz3yofskym2SimXswbSXuK7QSR1oHnBLBRMdamnVHLpSKQZhJUDejg==", + "resolved": "1.6.0", + "contentHash": "EycyMsb6rD2PK9P0SyibFfEhvWWttdrYhyPF4f41uzdB/44yQlV+2Wehxyg489Rj6gbPvSPgbKq0xsHJBhipZA==", "dependencies": { - "Azure.Core": "1.14.0", - "Microsoft.Identity.Client": "4.30.1", - "Microsoft.Identity.Client.Extensions.Msal": "2.18.4", + "Azure.Core": "1.24.0", + "Microsoft.Identity.Client": "4.39.0", + "Microsoft.Identity.Client.Extensions.Msal": "2.19.3", "System.Memory": "4.5.4", - "System.Security.Cryptography.ProtectedData": "4.5.0", - "System.Text.Json": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Security.Cryptography.ProtectedData": "4.7.0", + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Azure.Storage.Blobs": { @@ -116,8 +114,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.4", - "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", + "resolved": "5.2.8", + "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -125,93 +123,93 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", + "resolved": "2.2.0", + "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", + "resolved": "2.2.0", + "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", + "resolved": "2.2.0", + "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", + "resolved": "2.2.0", + "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Authorization": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Authorization": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", + "resolved": "2.2.0", + "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", + "resolved": "2.2.0", + "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Features": "2.2.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "kQUEVOU4loc8CPSb2WoHFTESqwIa8Ik7ysCBfTwzHAd0moWovc9JQLmhDIHlYLjHbyexqZAlkq/FPRUZqokebw==", + "resolved": "2.2.0", + "contentHash": "Nxs7Z1q3f1STfLYKJSVXCs1iBl+Ya6E8o4Oy1bCxJ/rNI44E/0f6tbsrVqAWfB7jlnJfyaAtIalBVxPKUPQb4Q==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.1", + "Microsoft.AspNetCore.Http.Features": "2.2.0", "System.Text.Encodings.Web": "4.5.0" } }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", + "resolved": "2.2.0", + "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Buffers": "4.5.0" } }, "Microsoft.AspNetCore.Http.Features": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VklZ7hWgSvHBcDtwYYkdMdI/adlf7ebxTZ9kdzAhX+gUs5jSHE9mZlTamdgf9miSsxc1QjNazHXTDJdVPZKKTw==", + "resolved": "2.2.0", + "contentHash": "ziFz5zH8f33En4dX81LW84I6XrYXKf9jg6aM39cM+LffN9KJahViKZ61dGMSO2gd3e+qe5yBRwsesvyqlZaSMg==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", + "resolved": "2.2.0", + "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -219,88 +217,89 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", + "resolved": "2.2.0", + "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.1.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Extensions.DependencyInjection": "2.1.0", + "resolved": "2.2.0", + "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.2.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection": "2.2.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.0" + "System.Threading.Tasks.Extensions": "4.5.1" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", + "resolved": "2.2.0", + "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.1.0", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0" + "Microsoft.AspNetCore.JsonPatch": "2.2.0", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", + "resolved": "2.2.0", + "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.WebUtilities": "2.1.0" + "Microsoft.AspNet.WebApi.Client": "5.2.6", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", + "resolved": "2.2.0", + "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", + "resolved": "2.2.2", + "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.ObjectPool": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", + "resolved": "2.2.0", + "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.WebUtilities": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "PGKIZt4+412Z/XPoSjvYu/QIbTxcAQuEFNoA1Pw8a9mgmO0ZhNBmfaNyhgXFf7Rq62kP0tT/2WXpxdcQhkFUPA==", + "resolved": "2.2.0", + "contentHash": "9ErxAAKaDzxXASB/b5uLEkLgUWv1QbeVxyJYEHQwMaxXOeFFVkQxiq8RyfVcifLU7NR0QY0p3acqx4ZpYfhHDg==", "dependencies": { - "Microsoft.Net.Http.Headers": "2.1.1", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Text.Encodings.Web": "4.5.0" } }, @@ -311,8 +310,8 @@ }, "Microsoft.Azure.WebJobs.Core": { "type": "Transitive", - "resolved": "3.0.31", - "contentHash": "iStV0MQ9env8R2F+8cbaynMK4TDkU6bpPVLdOzIs83iriHYMF+uWF6WZWI8ZJahWR37puQWc86u1YCsoIZEocA==", + "resolved": "3.0.32", + "contentHash": "pW5lyF0Tno1cC2VkmBLyv7E3o5ObDdbn3pfpUpKdksJo9ysCdQTpgc0Ib99wPHca6BgvoglicGbDYXuatanMfg==", "dependencies": { "System.ComponentModel.Annotations": "4.4.0", "System.Diagnostics.TraceSource": "4.3.0" @@ -330,15 +329,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.0.2", - "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", + "resolved": "3.2.0", + "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Azure.WebJobs": "3.0.2" + "Microsoft.AspNet.WebApi.Client": "5.2.8", + "Microsoft.AspNetCore.Http": "2.2.2", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.2", + "Microsoft.Azure.WebJobs": "3.0.32" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -373,8 +372,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "1.0.0", - "contentHash": "K63Y4hORbBcKLWH5wnKgzyn7TOfYzevIEwIedQHBIkmkEBA9SCqgvom+XTuE+fAFGvINGkhFItaZ2dvMGdT5iw==" + "resolved": "1.1.1", + "contentHash": "yuvf07qFWFqtK3P/MRkEKLhn5r2UbSpVueRziSqj0yJQIKFwG1pq9mOayK3zE5qZCTs0CbrwL9M6R8VwqyGy2w==" }, "Microsoft.CodeAnalysis.Analyzers": { "type": "Transitive", @@ -448,8 +447,8 @@ }, "Microsoft.Data.SqlClient.SNI.runtime": { "type": "Transitive", - "resolved": "3.0.0", - "contentHash": "n1sNyjJgu2pYWKgw3ZPikw3NiRvG4kt7Ya5MK8u77Rgj/1bTFqO/eDF4k5W9H5GXplMZCpKkNbp5kNBICgSB0w==" + "resolved": "5.0.1", + "contentHash": "y0X5MxiNdbITJYoafJ2ruaX6hqO0twpCGR/ipiDOe85JKLU8WL4TuAQfDe5qtt3bND5Je26HnrarLSAMMnVTNg==" }, "Microsoft.Diagnostics.NETCore.Client": { "type": "Transitive", @@ -508,10 +507,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", + "resolved": "2.2.0", + "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -551,16 +550,16 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "MgYpU5cwZohUMKKg3sbPhvGG+eAZ/59E9UwPwlrUkyXU+PGzqwZg9yyQNjhxuAWmoNoFReoemeCku50prYSGzA==" + "resolved": "2.2.0", + "contentHash": "f9hstgjVmr6rmrfGSpfsVOl2irKAgr1QjrSi3FgnS7kulxband50f2brRLwySAQTADPZeTdow0mpSMcoAdadCw==" }, "Microsoft.Extensions.DependencyModel": { "type": "Transitive", @@ -576,10 +575,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", + "resolved": "2.2.0", + "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -610,13 +609,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", + "resolved": "2.2.0", + "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0" } }, "Microsoft.Extensions.Logging": { @@ -632,8 +631,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" + "resolved": "2.2.0", + "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -646,16 +645,17 @@ }, "Microsoft.Extensions.ObjectPool": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "SErON45qh4ogDp6lr6UvVmFYW0FERihW+IQ+2JyFv1PUyWktcJytFaWH5zarufJvZwhci7Rf1IyGXr9pVEadTw==" + "resolved": "2.2.0", + "contentHash": "gA8H7uQOnM5gb+L0uTNjViHYr+hRDqCdfugheGo/MxQnuHzmhhzCBTIPm19qL1z1Xe0NEMabfcOBGv9QghlZ8g==" }, "Microsoft.Extensions.Options": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "V7lXCU78lAbzaulCGFKojcCyG8RTJicEbiBkPJjFqiqXwndEBBIehdXRMWEVU3UtzQ1yDvphiWUL9th6/4gJ7w==", + "resolved": "2.2.0", + "contentHash": "UpZLNLBpIZ0GTebShui7xXYh6DmBHjWM8NxGxZbdQh/bPZ5e6YswqI+bru6BnEL5eWiOdodsXtEz3FROcgi/qg==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.1", - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.Primitives": "2.2.0", + "System.ComponentModel.Annotations": "4.5.0" } }, "Microsoft.Extensions.Options.ConfigurationExtensions": { @@ -671,8 +671,8 @@ }, "Microsoft.Extensions.Primitives": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "scJ1GZNIxMmjpENh0UZ8XCQ6vzr/LzeF9WvEA51Ix2OQGAs9WPgPu8ABVUdvpKPLuor/t05gm6menJK3PwqOXg==", + "resolved": "2.2.0", + "contentHash": "azyQtqbm4fSaDzZHD/J+V6oWMFaf2tWP4WEGIYePLCMw3+b2RQdj9ybgbQyjCshcitQKQ4lEDOZjmSlTTrHxUg==", "dependencies": { "System.Memory": "4.5.1", "System.Runtime.CompilerServices.Unsafe": "4.5.1" @@ -680,78 +680,94 @@ }, "Microsoft.Identity.Client": { "type": "Transitive", - "resolved": "4.30.1", - "contentHash": "xk8tJeGfB2yD3+d7a0DXyV7/HYyEG10IofUHYHoPYKmDbroi/j9t1BqSHgbq1nARDjg7m8Ki6e21AyNU7e/R4Q==" + "resolved": "4.45.0", + "contentHash": "ircobISCLWbtE5eEoLKU+ldfZ8O41vg4lcy38KRj/znH17jvBiAl8oxcyNp89CsuqE3onxIpn21Ca7riyDDrRw==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.18.0" + } }, "Microsoft.Identity.Client.Extensions.Msal": { "type": "Transitive", - "resolved": "2.18.4", - "contentHash": "HpG4oLwhQsy0ce7OWq9iDdLtJKOvKRStIKoSEOeBMKuohfuOWNDyhg8fMAJkpG/kFeoe4J329fiMHcJmmB+FPw==", + "resolved": "2.19.3", + "contentHash": "zVVZjn8aW7W79rC1crioDgdOwaFTQorsSO6RgVlDDjc7MvbEGz071wSNrjVhzR0CdQn6Sefx7Abf1o7vasmrLg==", "dependencies": { - "Microsoft.Identity.Client": "4.30.0", + "Microsoft.Identity.Client": "4.38.0", "System.Security.Cryptography.ProtectedData": "4.5.0" } }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "XeE6LQtD719Qs2IG7HDi1TSw9LIkDbJ33xFiOBoHbApVw/8GpIBCbW+t7RwOjErUDyXZvjhZliwRkkLb8Z1uzg==" + }, "Microsoft.IdentityModel.JsonWebTokens": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "+7JIww64PkMt7NWFxoe4Y/joeF7TAtA/fQ0b2GFGcagzB59sKkTt/sMZWR6aSZht5YC7SdHi3W6yM1yylRGJCQ==", + "resolved": "6.21.0", + "contentHash": "d3h1/BaMeylKTkdP6XwRCxuOoDJZ44V9xaXr6gl5QxmpnZGdoK3bySo3OQN8ehRLJHShb94ElLUvoXyglQtgAw==", "dependencies": { - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "Microsoft.IdentityModel.Logging": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "Rfh/p4MaN4gkmhPxwbu8IjrmoDncGfHHPh1sTnc0AcM/Oc39/fzC9doKNWvUAjzFb8LqA6lgZyblTrIsX/wDXg==" + "resolved": "6.21.0", + "contentHash": "tuEhHIQwvBEhMf8I50hy8FHmRSUkffDFP5EdLsSDV4qRcl2wvOPkQxYqEzWkh+ytW6sbdJGEXElGhmhDfAxAKg==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.21.0" + } }, "Microsoft.IdentityModel.Protocols": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "OJZx5nPdiH+MEkwCkbJrTAUiO/YzLe0VSswNlDxJsJD9bhOIdXHufh650pfm59YH1DNevp3/bXzukKrG57gA1w==", + "resolved": "6.21.0", + "contentHash": "0FqY5cTLQKtHrClzHEI+QxJl8OBT2vUiEQQB7UKk832JDiJJmetzYZ3AdSrPjN/3l3nkhByeWzXnhrX0JbifKg==", "dependencies": { - "Microsoft.IdentityModel.Logging": "6.8.0", - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.Logging": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "Microsoft.IdentityModel.Protocols.OpenIdConnect": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "X/PiV5l3nYYsodtrNMrNQIVlDmHpjQQ5w48E+o/D5H4es2+4niEyQf3l03chvZGWNzBRhfSstaXr25/Ye4AeYw==", + "resolved": "6.21.0", + "contentHash": "vtSKL7n6EnAsLyxmiviusm6LKrblT2ndnNqN6rvVq6iIHAnPCK9E2DkDx6h1Jrpy1cvbp40r0cnTg23nhEAGTA==", "dependencies": { - "Microsoft.IdentityModel.Protocols": "6.8.0", - "System.IdentityModel.Tokens.Jwt": "6.8.0" + "Microsoft.IdentityModel.Protocols": "6.21.0", + "System.IdentityModel.Tokens.Jwt": "6.21.0" } }, "Microsoft.IdentityModel.Tokens": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "gTqzsGcmD13HgtNePPcuVHZ/NXWmyV+InJgalW/FhWpII1D7V1k0obIseGlWMeA4G+tZfeGMfXr0klnWbMR/mQ==", + "resolved": "6.21.0", + "contentHash": "AAEHZvZyb597a+QJSmtxH3n2P1nIJGpZ4Q89GTenknRx6T6zyfzf592yW/jA5e8EHN4tNMjjXHQaYWEq5+L05w==", "dependencies": { "Microsoft.CSharp": "4.5.0", - "Microsoft.IdentityModel.Logging": "6.8.0", + "Microsoft.IdentityModel.Logging": "6.21.0", "System.Security.Cryptography.Cng": "4.5.0" } }, "Microsoft.Net.Http.Headers": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "lPNIphl8b2EuhOE9dMH6EZDmu7pS882O+HMi5BJNsigxHaWlBrYxZHFZgE18cyaPp6SSZcTkKkuzfjV/RRQKlA==", + "resolved": "2.2.0", + "contentHash": "iZNkjYqlo8sIOI0bQfpsSoMTmB/kyvmV2h225ihyZT33aTp48ZpF6qYnXxzSXmHt8DpBAwBTX+1s1UFLbYfZKg==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1", + "Microsoft.Extensions.Primitives": "2.2.0", "System.Buffers": "4.5.0" } }, "Microsoft.NETCore.Platforms": { "type": "Transitive", - "resolved": "3.1.0", - "contentHash": "z7aeg8oHln2CuNulfhiLYxCVMPEwBl3rzicjvIX+4sUuCwvXw5oXQEtbiU2c0z4qYL5L3Kmx0mMA/+t/SbY67w==" + "resolved": "5.0.0", + "contentHash": "VyPlqzH2wavqquTcYpkIIAQ6WdenuKoFN0BdYBbCWsclXacSOHNQn66Gt4z5NBqEYW0FAPm5rlvki9ZiCij5xQ==" }, "Microsoft.NETCore.Targets": { "type": "Transitive", "resolved": "1.1.0", "contentHash": "aOZA3BWfz9RXjpzt0sRJJMjAscAUm3Hoa4UWAfceV9UTYxgwZ1lZt5nO2myFf+/jetYQo4uTP7zS8sJY67BBxg==" }, + "Microsoft.SqlServer.Server": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "N4KeF3cpcm1PUHym1RmakkzfkEv3GRMyofVv40uXsQhCQeglr2OHNcUk2WOG51AKpGO8ynGpo9M/kFXSzghwug==" + }, "Microsoft.TestPlatform.ObjectModel": { "type": "Transitive", "resolved": "17.0.0", @@ -782,19 +798,19 @@ }, "Microsoft.Win32.Registry": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "KSrRMb5vNi0CWSGG1++id2ZOs/1QhRqROt+qgbEAdQuGjGrFcl4AOl4/exGPUYz2wUnU42nvJqon1T3U0kPXLA==", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", "dependencies": { - "System.Security.AccessControl": "4.7.0", - "System.Security.Principal.Windows": "4.7.0" + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" } }, "Microsoft.Win32.SystemEvents": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "mtVirZr++rq+XCDITMUdnETD59XoeMxSpLRIII7JRI6Yj0LEDiO1pPn0ktlnIj12Ix8bfvQqQDMMIF9wC98oCA==", + "resolved": "5.0.0", + "contentHash": "Bh6blKG8VAKvXiLe2L+sEsn62nc1Ij34MrNxepD2OCrS5cpCwQa9MeLyhVQPQ/R4Wlzwuy6wMK8hLb11QPDRsQ==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0" + "Microsoft.NETCore.Platforms": "5.0.0" } }, "ncrontab.signed": { @@ -1077,8 +1093,8 @@ }, "System.ComponentModel.Annotations": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "29K3DQ+IGU7LBaMjTo7SI7T7X/tsMtLvz1p56LJ556Iu0Dw3pKZw5g8yCYCWMRxrOF0Hr0FU0FwW0o42y2sb3A==" + "resolved": "4.5.0", + "contentHash": "UxYQ3FGUOtzJ7LfSdnYSFd7+oEv6M8NgUatatIN2HxNtDdlcvFAf+VIq4Of9cDMJEJC0aSRv/x898RYhB4Yppg==" }, "System.ComponentModel.Primitives": { "type": "Transitive", @@ -1114,11 +1130,11 @@ }, "System.Configuration.ConfigurationManager": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "/anOTeSZCNNI2zDilogWrZ8pNqCmYbzGNexUnNhjW8k0sHqEZ2nHJBp147jBV3hGYswu5lINpNg1vxR7bnqvVA==", + "resolved": "5.0.0", + "contentHash": "aM7cbfEfVNlEEOj3DsZP+2g9NRwbkyiAv2isQEzw7pnkDg9ekCU2m1cdJLM02Uq691OaCS91tooaxcEn8d0q5w==", "dependencies": { - "System.Security.Cryptography.ProtectedData": "4.7.0", - "System.Security.Permissions": "4.7.0" + "System.Security.Cryptography.ProtectedData": "5.0.0", + "System.Security.Permissions": "5.0.0" } }, "System.Console": { @@ -1145,8 +1161,8 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "oJjw3uFuVDJiJNbCD8HB4a2p3NYLdt1fiT5OGsPLw+WTOuG0KpP4OXelMmmVKpClueMsit6xOlzy4wNKQFiBLg==" + "resolved": "5.0.0", + "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==" }, "System.Diagnostics.FileVersionInfo": { "type": "Transitive", @@ -1211,15 +1227,6 @@ "System.Runtime": "4.3.0" } }, - "System.Drawing.Common": { - "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "v+XbyYHaZjDfn0ENmJEV1VYLgGgCTx1gnfOBcppowbpOAriglYgGCvFCPr2EEZyBvXlpxbEsTwkOlInl107ahA==", - "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0", - "Microsoft.Win32.SystemEvents": "4.7.0" - } - }, "System.Dynamic.Runtime": { "type": "Transitive", "resolved": "4.3.0", @@ -1241,6 +1248,11 @@ "System.Threading": "4.3.0" } }, + "System.Formats.Asn1": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "MTvUIktmemNB+El0Fgw9egyqT9AYSIk6DTJeoDSpc3GIHxHCMo8COqkWT1mptX5tZ1SlQ6HJZ0OsSvMth1c12w==" + }, "System.Globalization": { "type": "Transitive", "resolved": "4.3.0", @@ -1277,11 +1289,11 @@ }, "System.IdentityModel.Tokens.Jwt": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "5tBCjAub2Bhd5qmcd0WhR5s354e4oLYa//kOWrkX+6/7ZbDDJjMTfwLSOiZ/MMpWdE4DWPLOfTLOq/juj9CKzA==", + "resolved": "6.21.0", + "contentHash": "JRD8AuypBE+2zYxT3dMJomQVsPYsCqlyZhWel3J1d5nzQokSRyTueF+Q4ID3Jcu6zSZKuzOdJ1MLTkbQsDqcvQ==", "dependencies": { - "Microsoft.IdentityModel.JsonWebTokens": "6.8.0", - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "System.IO": { @@ -1667,11 +1679,11 @@ }, "System.Security.AccessControl": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "JECvTt5aFF3WT3gHpfofL2MNNP6v84sxtXxpqhLBCcDRzqsPBmHhQ6shv4DwwN2tRlzsUxtb3G9M3763rbXKDg==", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0", - "System.Security.Principal.Windows": "4.7.0" + "Microsoft.NETCore.Platforms": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" } }, "System.Security.Cryptography.Algorithms": { @@ -1697,8 +1709,11 @@ }, "System.Security.Cryptography.Cng": { "type": "Transitive", - "resolved": "4.5.0", - "contentHash": "WG3r7EyjUe9CMPFSs6bty5doUqT+q9pbI80hlNzo2SkPkZ4VTuZkGWjpp77JB8+uaL4DFPRdBsAY+DX3dBK92A==" + "resolved": "5.0.0", + "contentHash": "jIMXsKn94T9JY7PvPq/tMfqa6GAaHpElRDpmG+SuL+D3+sTw2M8VhnibKnN8Tq+4JqbPJ/f+BwtLeDMEnzAvRg==", + "dependencies": { + "System.Formats.Asn1": "5.0.0" + } }, "System.Security.Cryptography.Csp": { "type": "Transitive", @@ -1775,8 +1790,8 @@ }, "System.Security.Cryptography.ProtectedData": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "ehYW0m9ptxpGWvE4zgqongBVWpSDU/JCFD4K7krxkQwSz/sFQjEXCUqpvencjy6DYDbn7Ig09R8GFffu8TtneQ==" + "resolved": "5.0.0", + "contentHash": "HGxMSAFAPLNoxBvSfW08vHde0F9uh7BjASwu6JF9JnXuEPhCY3YUqURn0+bQV/4UWeaqymmrHWV+Aw9riQCtCA==" }, "System.Security.Cryptography.X509Certificates": { "type": "Transitive", @@ -1812,17 +1827,17 @@ }, "System.Security.Permissions": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "dkOV6YYVBnYRa15/yv004eCGRBVADXw8qRbbNiCn/XpdJSUXkkUeIvdvFHkvnko4CdKMqG8yRHC4ox83LSlMsQ==", + "resolved": "5.0.0", + "contentHash": "uE8juAhEkp7KDBCdjDIE3H9R1HJuEHqeqX8nLX9gmYKWwsqk3T5qZlPx8qle5DPKimC/Fy3AFTdV7HamgCh9qQ==", "dependencies": { - "System.Security.AccessControl": "4.7.0", - "System.Windows.Extensions": "4.7.0" + "System.Security.AccessControl": "5.0.0", + "System.Windows.Extensions": "5.0.0" } }, "System.Security.Principal.Windows": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "ojD0PX0XhneCsUbAZVKdb7h/70vyYMDYs85lwEI+LngEONe/17A0cFaRFqZU+sOEidcVswYWikYOQ9PPfjlbtQ==" + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" }, "System.Text.Encoding": { "type": "Transitive", @@ -1836,10 +1851,10 @@ }, "System.Text.Encoding.CodePages": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "aeu4FlaUTemuT1qOd1MyU4T516QR4Fy+9yDbwWMPHOHy7U8FD6SgTzdZFO7gHcfAPHtECqInbwklVvUK4RHcNg==", + "resolved": "5.0.0", + "contentHash": "NyscU59xX6Uo91qvhOs2Ccho3AR2TnZPomo1Z0K6YpyztBPM/A5VbkzOO19sy3A3i1TtEnTxA7bCe3Us+r5MWg==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0" + "Microsoft.NETCore.Platforms": "5.0.0" } }, "System.Text.Encoding.Extensions": { @@ -1860,8 +1875,8 @@ }, "System.Text.Json": { "type": "Transitive", - "resolved": "4.6.0", - "contentHash": "4F8Xe+JIkVoDJ8hDAZ7HqLkjctN/6WItJIzQaifBwClC7wmoLSda/Sv2i6i1kycqDb3hWF4JCVbpAweyOKHEUA==" + "resolved": "4.7.2", + "contentHash": "TcMd95wcrubm9nHvJEQs70rC0H/8omiSGGpU4FQ/ZA1URIqD4pjmFJh2Mfv1yH1eHgJDWTi2hMDXwTET+zOOyg==" }, "System.Text.RegularExpressions": { "type": "Transitive", @@ -1897,8 +1912,8 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.2", - "contentHash": "BG/TNxDFv0svAzx8OiMXDlsHfGw623BZ8tCXw4YLhDFDvDhNUEV58jKYMGRnkbJNm7c3JNNJDiN7JBMzxRBR2w==" + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" }, "System.Threading.Tasks.Parallel": { "type": "Transitive", @@ -1940,10 +1955,10 @@ }, "System.Windows.Extensions": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "CeWTdRNfRaSh0pm2gDTJFwVaXfTq6Xwv/sA887iwPTneW7oMtMlpvDIO+U60+3GWTB7Aom6oQwv5VZVUhQRdPQ==", + "resolved": "5.0.0", + "contentHash": "c1ho9WU9ZxMZawML+ssPKZfdnrg/OjR3pe0m9v8230z3acqphwvPJqzAkH54xRYm5ntZHGG1EPP3sux9H3qSPg==", "dependencies": { - "System.Drawing.Common": "4.7.0" + "System.Drawing.Common": "5.0.0" } }, "System.Xml.ReaderWriter": { @@ -2088,69 +2103,68 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.14.0, )", - "Microsoft.AspNetCore.Http": "[2.1.22, )", - "Microsoft.Azure.WebJobs": "[3.0.31, )", - "Microsoft.Data.SqlClient": "[3.0.1, )", - "Newtonsoft.Json": "[11.0.2, )", - "System.Runtime.Caching": "[4.7.0, )", + "Microsoft.ApplicationInsights": "[2.17.0, )", + "Microsoft.AspNetCore.Http": "[2.2.2, )", + "Microsoft.Azure.WebJobs": "[3.0.32, )", + "Microsoft.Data.SqlClient": "[5.0.1, )", + "Newtonsoft.Json": "[13.0.1, )", + "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } }, "microsoft.azure.webjobs.extensions.sql.samples": { "type": "Project", "dependencies": { - "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )", - "Microsoft.NET.Sdk.Functions": "[3.1.1, )", - "Newtonsoft.Json": "[11.0.2, )" + "Microsoft.NET.Sdk.Functions": "[4.1.3, )", + "Newtonsoft.Json": "[13.0.1, )" } }, "microsoft.azure.webjobs.extensions.sql.tests": { "type": "Project", "dependencies": { - "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Sql.Samples": "[1.0.0, )", - "Microsoft.NET.Sdk.Functions": "[3.1.1, )", + "Microsoft.NET.Sdk.Functions": "[4.1.3, )", "Microsoft.NET.Test.Sdk": "[17.0.0, )", "Moq": "[4.14.3, )", - "Newtonsoft.Json": "[11.0.2, )", + "Newtonsoft.Json": "[13.0.1, )", "xunit": "[2.4.0, )", "xunit.runner.visualstudio": "[2.4.0, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.14.0, )", - "resolved": "2.14.0", - "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", + "requested": "[2.17.0, )", + "resolved": "2.17.0", + "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4" + "System.Diagnostics.DiagnosticSource": "5.0.0" } }, "Microsoft.AspNetCore.Http": { "type": "CentralTransitive", - "requested": "[2.1.22, )", - "resolved": "2.1.22", - "contentHash": "+Blk++1JWqghbl8+3azQmKhiNZA5wAepL9dY2I6KVmu2Ri07MAcvAVC888qUvO7yd7xgRgZOMfihezKg14O/2A==", + "requested": "[2.2.2, )", + "resolved": "2.2.2", + "contentHash": "BAibpoItxI5puk7YJbIGj95arZueM8B8M5xT1fXBn3hb3L2G3ucrZcYXv1gXdaroLbntUs8qeV8iuBrpjQsrKw==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.1", - "Microsoft.AspNetCore.WebUtilities": "2.1.1", - "Microsoft.Extensions.ObjectPool": "2.1.1", - "Microsoft.Extensions.Options": "2.1.1", - "Microsoft.Net.Http.Headers": "2.1.1" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.Azure.WebJobs": { "type": "CentralTransitive", - "requested": "[3.0.31, )", - "resolved": "3.0.31", - "contentHash": "Jn6E7OgT7LkwVB6lCpjXJcoQIvKrbJT+taVLA4CekEpa21pzZv6nQ2sYRSNzPz5ul3FAcYhmrCQgV7v2iopjgA==", + "requested": "[3.0.32, )", + "resolved": "3.0.32", + "contentHash": "uN8GsFqPFHHcSrwwj/+0tGe6F6cOwugqUiePPw7W3TL9YC594+Hw8GBK5S/fcDWXacqvRRGf9nDX8xP94/Yiyw==", "dependencies": { - "Microsoft.Azure.WebJobs.Core": "3.0.31", + "Microsoft.Azure.WebJobs.Core": "3.0.32", "Microsoft.Extensions.Configuration": "2.1.1", "Microsoft.Extensions.Configuration.Abstractions": "2.1.1", "Microsoft.Extensions.Configuration.EnvironmentVariables": "2.1.0", @@ -2176,45 +2190,50 @@ }, "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": { "type": "CentralTransitive", - "requested": "[1.2.3, )", - "resolved": "1.2.2", - "contentHash": "vpiNt3JM1pt/WrDIkg7G2DHhIpI4t5I+R9rmXCxIGiby5oPGEolyfiYZdEf2kMMN3SbWzVAbk4Q3jKgFhO9MaQ==", + "requested": "[4.0.1, )", + "resolved": "4.0.1", + "contentHash": "o1E0hetLv8Ix0teA1hGH9D136RGSs24Njm5+a4FKzJHLlxfclvmOxmcg87vcr6LIszKzenNKd1oJGnOwg2WMnw==", "dependencies": { "System.Runtime.Loader": "4.3.0" } }, "Microsoft.Data.SqlClient": { "type": "CentralTransitive", - "requested": "[3.0.1, )", - "resolved": "3.0.1", - "contentHash": "5Jgcds8yukUeOHvc8S0rGW87rs2uYEM9/YyrYIb/0C+vqzIa2GiqbVPCDVcnApWhs67OSXLTM7lO6jro24v/rA==", - "dependencies": { - "Azure.Identity": "1.3.0", - "Microsoft.Data.SqlClient.SNI.runtime": "3.0.0", - "Microsoft.Identity.Client": "4.22.0", - "Microsoft.IdentityModel.JsonWebTokens": "6.8.0", - "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.8.0", - "Microsoft.Win32.Registry": "4.7.0", - "System.Configuration.ConfigurationManager": "4.7.0", - "System.Diagnostics.DiagnosticSource": "4.7.0", - "System.Runtime.Caching": "4.7.0", - "System.Security.Principal.Windows": "4.7.0", - "System.Text.Encoding.CodePages": "4.7.0", + "requested": "[5.0.1, )", + "resolved": "5.0.1", + "contentHash": "uu8dfrsx081cSbEevWuZAvqdmANDGJkbLBL2G3j0LAZxX1Oy8RCVAaC4Lcuak6jNicWP6CWvHqBTIEmQNSxQlw==", + "dependencies": { + "Azure.Identity": "1.6.0", + "Microsoft.Data.SqlClient.SNI.runtime": "5.0.1", + "Microsoft.Identity.Client": "4.45.0", + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.21.0", + "Microsoft.SqlServer.Server": "1.0.0", + "Microsoft.Win32.Registry": "5.0.0", + "System.Buffers": "4.5.1", + "System.Configuration.ConfigurationManager": "5.0.0", + "System.Diagnostics.DiagnosticSource": "5.0.0", + "System.IO": "4.3.0", + "System.Resources.ResourceManager": "4.3.0", + "System.Runtime.Caching": "5.0.0", + "System.Security.Cryptography.Cng": "5.0.0", + "System.Security.Principal.Windows": "5.0.0", + "System.Text.Encoding.CodePages": "5.0.0", "System.Text.Encodings.Web": "4.7.2" } }, "Microsoft.NET.Sdk.Functions": { "type": "CentralTransitive", - "requested": "[3.1.1, )", - "resolved": "3.1.1", - "contentHash": "sPPLAjDYroeuIDKwff5B1XrwvGzJm9K9GabXurmfpaXa3M4POy7ngLcG5mm+2pwjTA7e870pIjt1N2DqVQS4yA==", + "requested": "[4.1.3, )", + "resolved": "4.1.3", + "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", - "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "1.2.2", - "Newtonsoft.Json": "11.0.2" + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", + "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", + "Newtonsoft.Json": "13.0.1" } }, "Microsoft.NET.Test.Sdk": { @@ -2245,17 +2264,26 @@ }, "Newtonsoft.Json": { "type": "CentralTransitive", - "requested": "[11.0.2, )", - "resolved": "11.0.2", - "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + }, + "System.Drawing.Common": { + "type": "CentralTransitive", + "requested": "[5.0.3, )", + "resolved": "5.0.0", + "contentHash": "SztFwAnpfKC8+sEKXAFxCBWhKQaEd97EiOL7oZJZP56zbqnLpmxACWA8aGseaUExciuEAUuR9dY8f7HkTRAdnw==", + "dependencies": { + "Microsoft.Win32.SystemEvents": "5.0.0" + } }, "System.Runtime.Caching": { "type": "CentralTransitive", - "requested": "[4.7.0, )", - "resolved": "4.7.0", - "contentHash": "NdvNRjTPxYvIEhXQszT9L9vJhdQoX6AQ0AlhjTU+5NqFQVuacJTfhPVAvtGWNA2OJCqRiR/okBcZgMwI6MqcZg==", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "30D6MkO8WF9jVGWZIP0hmCN8l9BTY4LCsAzLIe4xFSXzs+AjDotR7DpSmj27pFskDURzUvqYYY0ikModgBTxWw==", "dependencies": { - "System.Configuration.ConfigurationManager": "4.7.0" + "System.Configuration.ConfigurationManager": "5.0.0" } }, "xunit": { diff --git a/samples/samples-csharp/Microsoft.Azure.WebJobs.Extensions.Sql.Samples.csproj b/samples/samples-csharp/Microsoft.Azure.WebJobs.Extensions.Sql.Samples.csproj index 29224bf6b..3d1075333 100644 --- a/samples/samples-csharp/Microsoft.Azure.WebJobs.Extensions.Sql.Samples.csproj +++ b/samples/samples-csharp/Microsoft.Azure.WebJobs.Extensions.Sql.Samples.csproj @@ -1,7 +1,7 @@  - v3 + v4 diff --git a/samples/samples-csharp/OutputBindingSamples/AddProductWithIdentityColumnIncluded.cs b/samples/samples-csharp/OutputBindingSamples/AddProductWithIdentityColumnIncluded.cs index 664490f3f..7f68d7fca 100644 --- a/samples/samples-csharp/OutputBindingSamples/AddProductWithIdentityColumnIncluded.cs +++ b/samples/samples-csharp/OutputBindingSamples/AddProductWithIdentityColumnIncluded.cs @@ -30,7 +30,7 @@ public static IActionResult Run( product = new ProductWithOptionalId { Name = req.Query["name"], - ProductID = string.IsNullOrEmpty(req.Query["productId"]) ? null : (int?)int.Parse(req.Query["productId"]), + ProductID = string.IsNullOrEmpty(req.Query["productId"]) ? null : int.Parse(req.Query["productId"]), Cost = int.Parse(req.Query["cost"]) }; return new CreatedResult($"/api/addproductwithidentitycolumnincluded", product); diff --git a/samples/samples-csharp/global.json b/samples/samples-csharp/global.json deleted file mode 100644 index 1538a55f1..000000000 --- a/samples/samples-csharp/global.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sdk": { - "version": "3.1.419", - "rollForward": "latestFeature" - } -} \ No newline at end of file diff --git a/samples/samples-csharp/packages.lock.json b/samples/samples-csharp/packages.lock.json index a7ba791dc..6cc85cff0 100644 --- a/samples/samples-csharp/packages.lock.json +++ b/samples/samples-csharp/packages.lock.json @@ -1,18 +1,18 @@ { "version": 2, "dependencies": { - ".NETCoreApp,Version=v3.1": { + "net6.0": { "Microsoft.AspNetCore.Http": { "type": "Direct", - "requested": "[2.1.22, )", - "resolved": "2.1.22", - "contentHash": "+Blk++1JWqghbl8+3azQmKhiNZA5wAepL9dY2I6KVmu2Ri07MAcvAVC888qUvO7yd7xgRgZOMfihezKg14O/2A==", + "requested": "[2.2.2, )", + "resolved": "2.2.2", + "contentHash": "BAibpoItxI5puk7YJbIGj95arZueM8B8M5xT1fXBn3hb3L2G3ucrZcYXv1gXdaroLbntUs8qeV8iuBrpjQsrKw==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.1", - "Microsoft.AspNetCore.WebUtilities": "2.1.1", - "Microsoft.Extensions.ObjectPool": "2.1.1", - "Microsoft.Extensions.Options": "2.1.1", - "Microsoft.Net.Http.Headers": "2.1.1" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.Azure.WebJobs.Extensions.Storage": { @@ -27,52 +27,50 @@ }, "Microsoft.NET.Sdk.Functions": { "type": "Direct", - "requested": "[3.1.1, )", - "resolved": "3.1.1", - "contentHash": "sPPLAjDYroeuIDKwff5B1XrwvGzJm9K9GabXurmfpaXa3M4POy7ngLcG5mm+2pwjTA7e870pIjt1N2DqVQS4yA==", + "requested": "[4.1.3, )", + "resolved": "4.1.3", + "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", - "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "1.2.2", - "Newtonsoft.Json": "11.0.2" + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", + "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", + "Newtonsoft.Json": "13.0.1" } }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[11.0.2, )", - "resolved": "11.0.2", - "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" }, "Azure.Core": { "type": "Transitive", - "resolved": "1.19.0", - "contentHash": "lcDjG635DPE4fU5tqSueVMmzrx0QrIfPuY0+y6evHN5GanQ0GB+/4nuMHMmoNPwEow6OUPkJu4cZQxfHJQXPdA==", + "resolved": "1.24.0", + "contentHash": "+/qI1j2oU1S4/nvxb2k/wDsol00iGf1AyJX5g3epV7eOpQEP/2xcgh/cxgKMeFgn3U2fmgSiBnQZdkV+l5y0Uw==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "1.0.0", - "System.Buffers": "4.5.1", + "Microsoft.Bcl.AsyncInterfaces": "1.1.1", "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4", "System.Memory.Data": "1.0.2", "System.Numerics.Vectors": "4.5.0", "System.Text.Encodings.Web": "4.7.2", - "System.Text.Json": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Azure.Identity": { "type": "Transitive", - "resolved": "1.4.0", - "contentHash": "vvjdoDQb9WQyLkD1Uo5KFbwlW7xIsDMihz3yofskym2SimXswbSXuK7QSR1oHnBLBRMdamnVHLpSKQZhJUDejg==", + "resolved": "1.6.0", + "contentHash": "EycyMsb6rD2PK9P0SyibFfEhvWWttdrYhyPF4f41uzdB/44yQlV+2Wehxyg489Rj6gbPvSPgbKq0xsHJBhipZA==", "dependencies": { - "Azure.Core": "1.14.0", - "Microsoft.Identity.Client": "4.30.1", - "Microsoft.Identity.Client.Extensions.Msal": "2.18.4", + "Azure.Core": "1.24.0", + "Microsoft.Identity.Client": "4.39.0", + "Microsoft.Identity.Client.Extensions.Msal": "2.19.3", "System.Memory": "4.5.4", - "System.Security.Cryptography.ProtectedData": "4.5.0", - "System.Text.Json": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Security.Cryptography.ProtectedData": "4.7.0", + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Azure.Storage.Blobs": { @@ -104,8 +102,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.4", - "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", + "resolved": "5.2.8", + "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -113,93 +111,93 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", + "resolved": "2.2.0", + "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", + "resolved": "2.2.0", + "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", + "resolved": "2.2.0", + "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", + "resolved": "2.2.0", + "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Authorization": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Authorization": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", + "resolved": "2.2.0", + "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", + "resolved": "2.2.0", + "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Features": "2.2.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "kQUEVOU4loc8CPSb2WoHFTESqwIa8Ik7ysCBfTwzHAd0moWovc9JQLmhDIHlYLjHbyexqZAlkq/FPRUZqokebw==", + "resolved": "2.2.0", + "contentHash": "Nxs7Z1q3f1STfLYKJSVXCs1iBl+Ya6E8o4Oy1bCxJ/rNI44E/0f6tbsrVqAWfB7jlnJfyaAtIalBVxPKUPQb4Q==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.1", + "Microsoft.AspNetCore.Http.Features": "2.2.0", "System.Text.Encodings.Web": "4.5.0" } }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", + "resolved": "2.2.0", + "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Buffers": "4.5.0" } }, "Microsoft.AspNetCore.Http.Features": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VklZ7hWgSvHBcDtwYYkdMdI/adlf7ebxTZ9kdzAhX+gUs5jSHE9mZlTamdgf9miSsxc1QjNazHXTDJdVPZKKTw==", + "resolved": "2.2.0", + "contentHash": "ziFz5zH8f33En4dX81LW84I6XrYXKf9jg6aM39cM+LffN9KJahViKZ61dGMSO2gd3e+qe5yBRwsesvyqlZaSMg==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", + "resolved": "2.2.0", + "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -207,88 +205,89 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", + "resolved": "2.2.0", + "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.1.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Extensions.DependencyInjection": "2.1.0", + "resolved": "2.2.0", + "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.2.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection": "2.2.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.0" + "System.Threading.Tasks.Extensions": "4.5.1" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", + "resolved": "2.2.0", + "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.1.0", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0" + "Microsoft.AspNetCore.JsonPatch": "2.2.0", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", + "resolved": "2.2.0", + "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.WebUtilities": "2.1.0" + "Microsoft.AspNet.WebApi.Client": "5.2.6", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", + "resolved": "2.2.0", + "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", + "resolved": "2.2.2", + "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.ObjectPool": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", + "resolved": "2.2.0", + "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.WebUtilities": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "PGKIZt4+412Z/XPoSjvYu/QIbTxcAQuEFNoA1Pw8a9mgmO0ZhNBmfaNyhgXFf7Rq62kP0tT/2WXpxdcQhkFUPA==", + "resolved": "2.2.0", + "contentHash": "9ErxAAKaDzxXASB/b5uLEkLgUWv1QbeVxyJYEHQwMaxXOeFFVkQxiq8RyfVcifLU7NR0QY0p3acqx4ZpYfhHDg==", "dependencies": { - "Microsoft.Net.Http.Headers": "2.1.1", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Text.Encodings.Web": "4.5.0" } }, @@ -299,8 +298,8 @@ }, "Microsoft.Azure.WebJobs.Core": { "type": "Transitive", - "resolved": "3.0.31", - "contentHash": "iStV0MQ9env8R2F+8cbaynMK4TDkU6bpPVLdOzIs83iriHYMF+uWF6WZWI8ZJahWR37puQWc86u1YCsoIZEocA==", + "resolved": "3.0.32", + "contentHash": "pW5lyF0Tno1cC2VkmBLyv7E3o5ObDdbn3pfpUpKdksJo9ysCdQTpgc0Ib99wPHca6BgvoglicGbDYXuatanMfg==", "dependencies": { "System.ComponentModel.Annotations": "4.4.0", "System.Diagnostics.TraceSource": "4.3.0" @@ -318,15 +317,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.0.2", - "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", + "resolved": "3.2.0", + "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Azure.WebJobs": "3.0.2" + "Microsoft.AspNet.WebApi.Client": "5.2.8", + "Microsoft.AspNetCore.Http": "2.2.2", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.2", + "Microsoft.Azure.WebJobs": "3.0.32" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -361,8 +360,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "1.0.0", - "contentHash": "K63Y4hORbBcKLWH5wnKgzyn7TOfYzevIEwIedQHBIkmkEBA9SCqgvom+XTuE+fAFGvINGkhFItaZ2dvMGdT5iw==" + "resolved": "1.1.1", + "contentHash": "yuvf07qFWFqtK3P/MRkEKLhn5r2UbSpVueRziSqj0yJQIKFwG1pq9mOayK3zE5qZCTs0CbrwL9M6R8VwqyGy2w==" }, "Microsoft.CSharp": { "type": "Transitive", @@ -371,8 +370,8 @@ }, "Microsoft.Data.SqlClient.SNI.runtime": { "type": "Transitive", - "resolved": "3.0.0", - "contentHash": "n1sNyjJgu2pYWKgw3ZPikw3NiRvG4kt7Ya5MK8u77Rgj/1bTFqO/eDF4k5W9H5GXplMZCpKkNbp5kNBICgSB0w==" + "resolved": "5.0.1", + "contentHash": "y0X5MxiNdbITJYoafJ2ruaX6hqO0twpCGR/ipiDOe85JKLU8WL4TuAQfDe5qtt3bND5Je26HnrarLSAMMnVTNg==" }, "Microsoft.DotNet.PlatformAbstractions": { "type": "Transitive", @@ -413,10 +412,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", + "resolved": "2.2.0", + "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -456,16 +455,16 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "MgYpU5cwZohUMKKg3sbPhvGG+eAZ/59E9UwPwlrUkyXU+PGzqwZg9yyQNjhxuAWmoNoFReoemeCku50prYSGzA==" + "resolved": "2.2.0", + "contentHash": "f9hstgjVmr6rmrfGSpfsVOl2irKAgr1QjrSi3FgnS7kulxband50f2brRLwySAQTADPZeTdow0mpSMcoAdadCw==" }, "Microsoft.Extensions.DependencyModel": { "type": "Transitive", @@ -481,10 +480,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", + "resolved": "2.2.0", + "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -515,13 +514,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", + "resolved": "2.2.0", + "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0" } }, "Microsoft.Extensions.Logging": { @@ -537,8 +536,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" + "resolved": "2.2.0", + "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -551,16 +550,17 @@ }, "Microsoft.Extensions.ObjectPool": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "SErON45qh4ogDp6lr6UvVmFYW0FERihW+IQ+2JyFv1PUyWktcJytFaWH5zarufJvZwhci7Rf1IyGXr9pVEadTw==" + "resolved": "2.2.0", + "contentHash": "gA8H7uQOnM5gb+L0uTNjViHYr+hRDqCdfugheGo/MxQnuHzmhhzCBTIPm19qL1z1Xe0NEMabfcOBGv9QghlZ8g==" }, "Microsoft.Extensions.Options": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "V7lXCU78lAbzaulCGFKojcCyG8RTJicEbiBkPJjFqiqXwndEBBIehdXRMWEVU3UtzQ1yDvphiWUL9th6/4gJ7w==", + "resolved": "2.2.0", + "contentHash": "UpZLNLBpIZ0GTebShui7xXYh6DmBHjWM8NxGxZbdQh/bPZ5e6YswqI+bru6BnEL5eWiOdodsXtEz3FROcgi/qg==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.1", - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.Primitives": "2.2.0", + "System.ComponentModel.Annotations": "4.5.0" } }, "Microsoft.Extensions.Options.ConfigurationExtensions": { @@ -576,8 +576,8 @@ }, "Microsoft.Extensions.Primitives": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "scJ1GZNIxMmjpENh0UZ8XCQ6vzr/LzeF9WvEA51Ix2OQGAs9WPgPu8ABVUdvpKPLuor/t05gm6menJK3PwqOXg==", + "resolved": "2.2.0", + "contentHash": "azyQtqbm4fSaDzZHD/J+V6oWMFaf2tWP4WEGIYePLCMw3+b2RQdj9ybgbQyjCshcitQKQ4lEDOZjmSlTTrHxUg==", "dependencies": { "System.Memory": "4.5.1", "System.Runtime.CompilerServices.Unsafe": "4.5.1" @@ -585,78 +585,94 @@ }, "Microsoft.Identity.Client": { "type": "Transitive", - "resolved": "4.30.1", - "contentHash": "xk8tJeGfB2yD3+d7a0DXyV7/HYyEG10IofUHYHoPYKmDbroi/j9t1BqSHgbq1nARDjg7m8Ki6e21AyNU7e/R4Q==" + "resolved": "4.45.0", + "contentHash": "ircobISCLWbtE5eEoLKU+ldfZ8O41vg4lcy38KRj/znH17jvBiAl8oxcyNp89CsuqE3onxIpn21Ca7riyDDrRw==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.18.0" + } }, "Microsoft.Identity.Client.Extensions.Msal": { "type": "Transitive", - "resolved": "2.18.4", - "contentHash": "HpG4oLwhQsy0ce7OWq9iDdLtJKOvKRStIKoSEOeBMKuohfuOWNDyhg8fMAJkpG/kFeoe4J329fiMHcJmmB+FPw==", + "resolved": "2.19.3", + "contentHash": "zVVZjn8aW7W79rC1crioDgdOwaFTQorsSO6RgVlDDjc7MvbEGz071wSNrjVhzR0CdQn6Sefx7Abf1o7vasmrLg==", "dependencies": { - "Microsoft.Identity.Client": "4.30.0", + "Microsoft.Identity.Client": "4.38.0", "System.Security.Cryptography.ProtectedData": "4.5.0" } }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "XeE6LQtD719Qs2IG7HDi1TSw9LIkDbJ33xFiOBoHbApVw/8GpIBCbW+t7RwOjErUDyXZvjhZliwRkkLb8Z1uzg==" + }, "Microsoft.IdentityModel.JsonWebTokens": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "+7JIww64PkMt7NWFxoe4Y/joeF7TAtA/fQ0b2GFGcagzB59sKkTt/sMZWR6aSZht5YC7SdHi3W6yM1yylRGJCQ==", + "resolved": "6.21.0", + "contentHash": "d3h1/BaMeylKTkdP6XwRCxuOoDJZ44V9xaXr6gl5QxmpnZGdoK3bySo3OQN8ehRLJHShb94ElLUvoXyglQtgAw==", "dependencies": { - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "Microsoft.IdentityModel.Logging": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "Rfh/p4MaN4gkmhPxwbu8IjrmoDncGfHHPh1sTnc0AcM/Oc39/fzC9doKNWvUAjzFb8LqA6lgZyblTrIsX/wDXg==" + "resolved": "6.21.0", + "contentHash": "tuEhHIQwvBEhMf8I50hy8FHmRSUkffDFP5EdLsSDV4qRcl2wvOPkQxYqEzWkh+ytW6sbdJGEXElGhmhDfAxAKg==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.21.0" + } }, "Microsoft.IdentityModel.Protocols": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "OJZx5nPdiH+MEkwCkbJrTAUiO/YzLe0VSswNlDxJsJD9bhOIdXHufh650pfm59YH1DNevp3/bXzukKrG57gA1w==", + "resolved": "6.21.0", + "contentHash": "0FqY5cTLQKtHrClzHEI+QxJl8OBT2vUiEQQB7UKk832JDiJJmetzYZ3AdSrPjN/3l3nkhByeWzXnhrX0JbifKg==", "dependencies": { - "Microsoft.IdentityModel.Logging": "6.8.0", - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.Logging": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "Microsoft.IdentityModel.Protocols.OpenIdConnect": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "X/PiV5l3nYYsodtrNMrNQIVlDmHpjQQ5w48E+o/D5H4es2+4niEyQf3l03chvZGWNzBRhfSstaXr25/Ye4AeYw==", + "resolved": "6.21.0", + "contentHash": "vtSKL7n6EnAsLyxmiviusm6LKrblT2ndnNqN6rvVq6iIHAnPCK9E2DkDx6h1Jrpy1cvbp40r0cnTg23nhEAGTA==", "dependencies": { - "Microsoft.IdentityModel.Protocols": "6.8.0", - "System.IdentityModel.Tokens.Jwt": "6.8.0" + "Microsoft.IdentityModel.Protocols": "6.21.0", + "System.IdentityModel.Tokens.Jwt": "6.21.0" } }, "Microsoft.IdentityModel.Tokens": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "gTqzsGcmD13HgtNePPcuVHZ/NXWmyV+InJgalW/FhWpII1D7V1k0obIseGlWMeA4G+tZfeGMfXr0klnWbMR/mQ==", + "resolved": "6.21.0", + "contentHash": "AAEHZvZyb597a+QJSmtxH3n2P1nIJGpZ4Q89GTenknRx6T6zyfzf592yW/jA5e8EHN4tNMjjXHQaYWEq5+L05w==", "dependencies": { "Microsoft.CSharp": "4.5.0", - "Microsoft.IdentityModel.Logging": "6.8.0", + "Microsoft.IdentityModel.Logging": "6.21.0", "System.Security.Cryptography.Cng": "4.5.0" } }, "Microsoft.Net.Http.Headers": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "lPNIphl8b2EuhOE9dMH6EZDmu7pS882O+HMi5BJNsigxHaWlBrYxZHFZgE18cyaPp6SSZcTkKkuzfjV/RRQKlA==", + "resolved": "2.2.0", + "contentHash": "iZNkjYqlo8sIOI0bQfpsSoMTmB/kyvmV2h225ihyZT33aTp48ZpF6qYnXxzSXmHt8DpBAwBTX+1s1UFLbYfZKg==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1", + "Microsoft.Extensions.Primitives": "2.2.0", "System.Buffers": "4.5.0" } }, "Microsoft.NETCore.Platforms": { "type": "Transitive", - "resolved": "3.1.0", - "contentHash": "z7aeg8oHln2CuNulfhiLYxCVMPEwBl3rzicjvIX+4sUuCwvXw5oXQEtbiU2c0z4qYL5L3Kmx0mMA/+t/SbY67w==" + "resolved": "5.0.0", + "contentHash": "VyPlqzH2wavqquTcYpkIIAQ6WdenuKoFN0BdYBbCWsclXacSOHNQn66Gt4z5NBqEYW0FAPm5rlvki9ZiCij5xQ==" }, "Microsoft.NETCore.Targets": { "type": "Transitive", "resolved": "1.1.0", "contentHash": "aOZA3BWfz9RXjpzt0sRJJMjAscAUm3Hoa4UWAfceV9UTYxgwZ1lZt5nO2myFf+/jetYQo4uTP7zS8sJY67BBxg==" }, + "Microsoft.SqlServer.Server": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "N4KeF3cpcm1PUHym1RmakkzfkEv3GRMyofVv40uXsQhCQeglr2OHNcUk2WOG51AKpGO8ynGpo9M/kFXSzghwug==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -669,19 +685,19 @@ }, "Microsoft.Win32.Registry": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "KSrRMb5vNi0CWSGG1++id2ZOs/1QhRqROt+qgbEAdQuGjGrFcl4AOl4/exGPUYz2wUnU42nvJqon1T3U0kPXLA==", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", "dependencies": { - "System.Security.AccessControl": "4.7.0", - "System.Security.Principal.Windows": "4.7.0" + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" } }, "Microsoft.Win32.SystemEvents": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "mtVirZr++rq+XCDITMUdnETD59XoeMxSpLRIII7JRI6Yj0LEDiO1pPn0ktlnIj12Ix8bfvQqQDMMIF9wC98oCA==", + "resolved": "5.0.0", + "contentHash": "Bh6blKG8VAKvXiLe2L+sEsn62nc1Ij34MrNxepD2OCrS5cpCwQa9MeLyhVQPQ/R4Wlzwuy6wMK8hLb11QPDRsQ==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0" + "Microsoft.NETCore.Platforms": "5.0.0" } }, "ncrontab.signed": { @@ -906,16 +922,16 @@ }, "System.ComponentModel.Annotations": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "29K3DQ+IGU7LBaMjTo7SI7T7X/tsMtLvz1p56LJ556Iu0Dw3pKZw5g8yCYCWMRxrOF0Hr0FU0FwW0o42y2sb3A==" + "resolved": "4.5.0", + "contentHash": "UxYQ3FGUOtzJ7LfSdnYSFd7+oEv6M8NgUatatIN2HxNtDdlcvFAf+VIq4Of9cDMJEJC0aSRv/x898RYhB4Yppg==" }, "System.Configuration.ConfigurationManager": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "/anOTeSZCNNI2zDilogWrZ8pNqCmYbzGNexUnNhjW8k0sHqEZ2nHJBp147jBV3hGYswu5lINpNg1vxR7bnqvVA==", + "resolved": "5.0.0", + "contentHash": "aM7cbfEfVNlEEOj3DsZP+2g9NRwbkyiAv2isQEzw7pnkDg9ekCU2m1cdJLM02Uq691OaCS91tooaxcEn8d0q5w==", "dependencies": { - "System.Security.Cryptography.ProtectedData": "4.7.0", - "System.Security.Permissions": "4.7.0" + "System.Security.Cryptography.ProtectedData": "5.0.0", + "System.Security.Permissions": "5.0.0" } }, "System.Console": { @@ -942,8 +958,8 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "oJjw3uFuVDJiJNbCD8HB4a2p3NYLdt1fiT5OGsPLw+WTOuG0KpP4OXelMmmVKpClueMsit6xOlzy4wNKQFiBLg==" + "resolved": "5.0.0", + "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==" }, "System.Diagnostics.Tools": { "type": "Transitive", @@ -981,15 +997,6 @@ "System.Runtime": "4.3.0" } }, - "System.Drawing.Common": { - "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "v+XbyYHaZjDfn0ENmJEV1VYLgGgCTx1gnfOBcppowbpOAriglYgGCvFCPr2EEZyBvXlpxbEsTwkOlInl107ahA==", - "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0", - "Microsoft.Win32.SystemEvents": "4.7.0" - } - }, "System.Dynamic.Runtime": { "type": "Transitive", "resolved": "4.0.11", @@ -1012,6 +1019,11 @@ "System.Threading": "4.0.11" } }, + "System.Formats.Asn1": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "MTvUIktmemNB+El0Fgw9egyqT9AYSIk6DTJeoDSpc3GIHxHCMo8COqkWT1mptX5tZ1SlQ6HJZ0OsSvMth1c12w==" + }, "System.Globalization": { "type": "Transitive", "resolved": "4.3.0", @@ -1048,11 +1060,11 @@ }, "System.IdentityModel.Tokens.Jwt": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "5tBCjAub2Bhd5qmcd0WhR5s354e4oLYa//kOWrkX+6/7ZbDDJjMTfwLSOiZ/MMpWdE4DWPLOfTLOq/juj9CKzA==", + "resolved": "6.21.0", + "contentHash": "JRD8AuypBE+2zYxT3dMJomQVsPYsCqlyZhWel3J1d5nzQokSRyTueF+Q4ID3Jcu6zSZKuzOdJ1MLTkbQsDqcvQ==", "dependencies": { - "Microsoft.IdentityModel.JsonWebTokens": "6.8.0", - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "System.IO": { @@ -1423,11 +1435,11 @@ }, "System.Security.AccessControl": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "JECvTt5aFF3WT3gHpfofL2MNNP6v84sxtXxpqhLBCcDRzqsPBmHhQ6shv4DwwN2tRlzsUxtb3G9M3763rbXKDg==", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0", - "System.Security.Principal.Windows": "4.7.0" + "Microsoft.NETCore.Platforms": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" } }, "System.Security.Cryptography.Algorithms": { @@ -1453,8 +1465,11 @@ }, "System.Security.Cryptography.Cng": { "type": "Transitive", - "resolved": "4.5.0", - "contentHash": "WG3r7EyjUe9CMPFSs6bty5doUqT+q9pbI80hlNzo2SkPkZ4VTuZkGWjpp77JB8+uaL4DFPRdBsAY+DX3dBK92A==" + "resolved": "5.0.0", + "contentHash": "jIMXsKn94T9JY7PvPq/tMfqa6GAaHpElRDpmG+SuL+D3+sTw2M8VhnibKnN8Tq+4JqbPJ/f+BwtLeDMEnzAvRg==", + "dependencies": { + "System.Formats.Asn1": "5.0.0" + } }, "System.Security.Cryptography.Csp": { "type": "Transitive", @@ -1531,8 +1546,8 @@ }, "System.Security.Cryptography.ProtectedData": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "ehYW0m9ptxpGWvE4zgqongBVWpSDU/JCFD4K7krxkQwSz/sFQjEXCUqpvencjy6DYDbn7Ig09R8GFffu8TtneQ==" + "resolved": "5.0.0", + "contentHash": "HGxMSAFAPLNoxBvSfW08vHde0F9uh7BjASwu6JF9JnXuEPhCY3YUqURn0+bQV/4UWeaqymmrHWV+Aw9riQCtCA==" }, "System.Security.Cryptography.X509Certificates": { "type": "Transitive", @@ -1568,17 +1583,17 @@ }, "System.Security.Permissions": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "dkOV6YYVBnYRa15/yv004eCGRBVADXw8qRbbNiCn/XpdJSUXkkUeIvdvFHkvnko4CdKMqG8yRHC4ox83LSlMsQ==", + "resolved": "5.0.0", + "contentHash": "uE8juAhEkp7KDBCdjDIE3H9R1HJuEHqeqX8nLX9gmYKWwsqk3T5qZlPx8qle5DPKimC/Fy3AFTdV7HamgCh9qQ==", "dependencies": { - "System.Security.AccessControl": "4.7.0", - "System.Windows.Extensions": "4.7.0" + "System.Security.AccessControl": "5.0.0", + "System.Windows.Extensions": "5.0.0" } }, "System.Security.Principal.Windows": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "ojD0PX0XhneCsUbAZVKdb7h/70vyYMDYs85lwEI+LngEONe/17A0cFaRFqZU+sOEidcVswYWikYOQ9PPfjlbtQ==" + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" }, "System.Text.Encoding": { "type": "Transitive", @@ -1592,10 +1607,10 @@ }, "System.Text.Encoding.CodePages": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "aeu4FlaUTemuT1qOd1MyU4T516QR4Fy+9yDbwWMPHOHy7U8FD6SgTzdZFO7gHcfAPHtECqInbwklVvUK4RHcNg==", + "resolved": "5.0.0", + "contentHash": "NyscU59xX6Uo91qvhOs2Ccho3AR2TnZPomo1Z0K6YpyztBPM/A5VbkzOO19sy3A3i1TtEnTxA7bCe3Us+r5MWg==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0" + "Microsoft.NETCore.Platforms": "5.0.0" } }, "System.Text.Encoding.Extensions": { @@ -1616,8 +1631,8 @@ }, "System.Text.Json": { "type": "Transitive", - "resolved": "4.6.0", - "contentHash": "4F8Xe+JIkVoDJ8hDAZ7HqLkjctN/6WItJIzQaifBwClC7wmoLSda/Sv2i6i1kycqDb3hWF4JCVbpAweyOKHEUA==" + "resolved": "4.7.2", + "contentHash": "TcMd95wcrubm9nHvJEQs70rC0H/8omiSGGpU4FQ/ZA1URIqD4pjmFJh2Mfv1yH1eHgJDWTi2hMDXwTET+zOOyg==" }, "System.Text.RegularExpressions": { "type": "Transitive", @@ -1653,8 +1668,8 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.2", - "contentHash": "BG/TNxDFv0svAzx8OiMXDlsHfGw623BZ8tCXw4YLhDFDvDhNUEV58jKYMGRnkbJNm7c3JNNJDiN7JBMzxRBR2w==" + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" }, "System.Threading.Timer": { "type": "Transitive", @@ -1668,10 +1683,10 @@ }, "System.Windows.Extensions": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "CeWTdRNfRaSh0pm2gDTJFwVaXfTq6Xwv/sA887iwPTneW7oMtMlpvDIO+U60+3GWTB7Aom6oQwv5VZVUhQRdPQ==", + "resolved": "5.0.0", + "contentHash": "c1ho9WU9ZxMZawML+ssPKZfdnrg/OjR3pe0m9v8230z3acqphwvPJqzAkH54xRYm5ntZHGG1EPP3sux9H3qSPg==", "dependencies": { - "System.Drawing.Common": "4.7.0" + "System.Drawing.Common": "5.0.0" } }, "System.Xml.ReaderWriter": { @@ -1727,32 +1742,31 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.14.0, )", - "Microsoft.AspNetCore.Http": "[2.1.22, )", - "Microsoft.Azure.WebJobs": "[3.0.31, )", - "Microsoft.Data.SqlClient": "[3.0.1, )", - "Newtonsoft.Json": "[11.0.2, )", - "System.Runtime.Caching": "[4.7.0, )", + "Microsoft.ApplicationInsights": "[2.17.0, )", + "Microsoft.AspNetCore.Http": "[2.2.2, )", + "Microsoft.Azure.WebJobs": "[3.0.32, )", + "Microsoft.Data.SqlClient": "[5.0.1, )", + "Newtonsoft.Json": "[13.0.1, )", + "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.14.0, )", - "resolved": "2.14.0", - "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", + "requested": "[2.17.0, )", + "resolved": "2.17.0", + "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4" + "System.Diagnostics.DiagnosticSource": "5.0.0" } }, "Microsoft.Azure.WebJobs": { "type": "CentralTransitive", - "requested": "[3.0.31, )", - "resolved": "3.0.31", - "contentHash": "Jn6E7OgT7LkwVB6lCpjXJcoQIvKrbJT+taVLA4CekEpa21pzZv6nQ2sYRSNzPz5ul3FAcYhmrCQgV7v2iopjgA==", + "requested": "[3.0.32, )", + "resolved": "3.0.32", + "contentHash": "uN8GsFqPFHHcSrwwj/+0tGe6F6cOwugqUiePPw7W3TL9YC594+Hw8GBK5S/fcDWXacqvRRGf9nDX8xP94/Yiyw==", "dependencies": { - "Microsoft.Azure.WebJobs.Core": "3.0.31", + "Microsoft.Azure.WebJobs.Core": "3.0.32", "Microsoft.Extensions.Configuration": "2.1.1", "Microsoft.Extensions.Configuration.Abstractions": "2.1.1", "Microsoft.Extensions.Configuration.EnvironmentVariables": "2.1.0", @@ -1768,30 +1782,35 @@ }, "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": { "type": "CentralTransitive", - "requested": "[1.2.3, )", - "resolved": "1.2.2", - "contentHash": "vpiNt3JM1pt/WrDIkg7G2DHhIpI4t5I+R9rmXCxIGiby5oPGEolyfiYZdEf2kMMN3SbWzVAbk4Q3jKgFhO9MaQ==", + "requested": "[4.0.1, )", + "resolved": "4.0.1", + "contentHash": "o1E0hetLv8Ix0teA1hGH9D136RGSs24Njm5+a4FKzJHLlxfclvmOxmcg87vcr6LIszKzenNKd1oJGnOwg2WMnw==", "dependencies": { "System.Runtime.Loader": "4.3.0" } }, "Microsoft.Data.SqlClient": { "type": "CentralTransitive", - "requested": "[3.0.1, )", - "resolved": "3.0.1", - "contentHash": "5Jgcds8yukUeOHvc8S0rGW87rs2uYEM9/YyrYIb/0C+vqzIa2GiqbVPCDVcnApWhs67OSXLTM7lO6jro24v/rA==", - "dependencies": { - "Azure.Identity": "1.3.0", - "Microsoft.Data.SqlClient.SNI.runtime": "3.0.0", - "Microsoft.Identity.Client": "4.22.0", - "Microsoft.IdentityModel.JsonWebTokens": "6.8.0", - "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.8.0", - "Microsoft.Win32.Registry": "4.7.0", - "System.Configuration.ConfigurationManager": "4.7.0", - "System.Diagnostics.DiagnosticSource": "4.7.0", - "System.Runtime.Caching": "4.7.0", - "System.Security.Principal.Windows": "4.7.0", - "System.Text.Encoding.CodePages": "4.7.0", + "requested": "[5.0.1, )", + "resolved": "5.0.1", + "contentHash": "uu8dfrsx081cSbEevWuZAvqdmANDGJkbLBL2G3j0LAZxX1Oy8RCVAaC4Lcuak6jNicWP6CWvHqBTIEmQNSxQlw==", + "dependencies": { + "Azure.Identity": "1.6.0", + "Microsoft.Data.SqlClient.SNI.runtime": "5.0.1", + "Microsoft.Identity.Client": "4.45.0", + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.21.0", + "Microsoft.SqlServer.Server": "1.0.0", + "Microsoft.Win32.Registry": "5.0.0", + "System.Buffers": "4.5.1", + "System.Configuration.ConfigurationManager": "5.0.0", + "System.Diagnostics.DiagnosticSource": "5.0.0", + "System.IO": "4.3.0", + "System.Resources.ResourceManager": "4.3.0", + "System.Runtime.Caching": "5.0.0", + "System.Security.Cryptography.Cng": "5.0.0", + "System.Security.Principal.Windows": "5.0.0", + "System.Text.Encoding.CodePages": "5.0.0", "System.Text.Encodings.Web": "4.7.2" } }, @@ -1801,13 +1820,22 @@ "resolved": "3.3.2", "contentHash": "MQc8GppZJLmjvcpEdf3EkC6ovsp7gRWt2e5mC7dcIOrgwSc+yjFd3JQ0iRqr3XrUT6rb/phv0IkEmBtbfVA7AQ==" }, + "System.Drawing.Common": { + "type": "CentralTransitive", + "requested": "[5.0.3, )", + "resolved": "5.0.0", + "contentHash": "SztFwAnpfKC8+sEKXAFxCBWhKQaEd97EiOL7oZJZP56zbqnLpmxACWA8aGseaUExciuEAUuR9dY8f7HkTRAdnw==", + "dependencies": { + "Microsoft.Win32.SystemEvents": "5.0.0" + } + }, "System.Runtime.Caching": { "type": "CentralTransitive", - "requested": "[4.7.0, )", - "resolved": "4.7.0", - "contentHash": "NdvNRjTPxYvIEhXQszT9L9vJhdQoX6AQ0AlhjTU+5NqFQVuacJTfhPVAvtGWNA2OJCqRiR/okBcZgMwI6MqcZg==", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "30D6MkO8WF9jVGWZIP0hmCN8l9BTY4LCsAzLIe4xFSXzs+AjDotR7DpSmj27pFskDURzUvqYYY0ikModgBTxWw==", "dependencies": { - "System.Configuration.ConfigurationManager": "4.7.0" + "System.Configuration.ConfigurationManager": "5.0.0" } } } diff --git a/src/SqlAsyncCollector.cs b/src/SqlAsyncCollector.cs index b232fc913..c14f2ce03 100644 --- a/src/SqlAsyncCollector.cs +++ b/src/SqlAsyncCollector.cs @@ -21,6 +21,7 @@ using Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry; using System.Diagnostics; using Newtonsoft.Json.Linq; +using static Microsoft.Azure.WebJobs.Extensions.Sql.SqlBindingConstants; namespace Microsoft.Azure.WebJobs.Extensions.Sql { @@ -377,8 +378,6 @@ private static void GenerateDataQueryForMerge(TableInformation table, IEnumerabl public class TableInformation { - private const string ISO_8061_DATETIME_FORMAT = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff"; - public IEnumerable PrimaryKeys { get; } /// diff --git a/src/SqlAsyncEnumerable.cs b/src/SqlAsyncEnumerable.cs index 9b10eab81..cfff1bfc0 100644 --- a/src/SqlAsyncEnumerable.cs +++ b/src/SqlAsyncEnumerable.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.Data.SqlClient; using Newtonsoft.Json; +using static Microsoft.Azure.WebJobs.Extensions.Sql.SqlBindingConstants; namespace Microsoft.Azure.WebJobs.Extensions.Sql { /// A user-defined POCO that represents a row of the user's table @@ -126,7 +127,11 @@ private async Task GetNextRowAsync() /// JSON string version of the SQL row private string SerializeRow() { - return JsonConvert.SerializeObject(SqlBindingUtilities.BuildDictionaryFromSqlRow(this._reader)); + var jsonSerializerSettings = new JsonSerializerSettings() + { + DateFormatString = ISO_8061_DATETIME_FORMAT + }; + return JsonConvert.SerializeObject(SqlBindingUtilities.BuildDictionaryFromSqlRow(this._reader), jsonSerializerSettings); } } } diff --git a/src/SqlAttribute.cs b/src/SqlAttribute.cs index a7a89aad4..877307c34 100644 --- a/src/SqlAttribute.cs +++ b/src/SqlAttribute.cs @@ -27,9 +27,9 @@ public SqlAttribute(string commandText) /// /// The name of the app setting where the SQL connection string is stored - /// (see https://docs.microsoft.com/en-us/dotnet/api/microsoft.data.sqlclient.sqlconnection?view=sqlclient-dotnet-core-2.0). + /// (see https://docs.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection). /// The attributes specified in the connection string are listed here - /// https://docs.microsoft.com/en-us/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring?view=sqlclient-dotnet-core-2.0 + /// https://docs.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlconnection.connectionstring /// For example, to create a connection to the "TestDB" located at the URL "test.database.windows.net" using a User ID and password, /// create a ConnectionStringSetting with a name like SqlServerAuthentication. The value of the SqlServerAuthentication app setting /// would look like "Data Source=test.database.windows.net;Database=TestDB;User ID={userid};Password={password}". diff --git a/src/SqlBindingConstants.cs b/src/SqlBindingConstants.cs new file mode 100644 index 000000000..cb91184b2 --- /dev/null +++ b/src/SqlBindingConstants.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.WebJobs.Extensions.Sql +{ + internal static class SqlBindingConstants + { + public const string ISO_8061_DATETIME_FORMAT = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffZ"; + } +} \ No newline at end of file diff --git a/src/SqlConverters.cs b/src/SqlConverters.cs index cc56425a7..ddb872a1c 100644 --- a/src/SqlConverters.cs +++ b/src/SqlConverters.cs @@ -14,6 +14,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Microsoft.Extensions.Logging; +using static Microsoft.Azure.WebJobs.Extensions.Sql.SqlBindingConstants; namespace Microsoft.Azure.WebJobs.Extensions.Sql { @@ -186,7 +187,12 @@ public virtual async Task BuildItemFromAttributeAsync(SqlAttribute attri var dataTable = new DataTable(); adapter.Fill(dataTable); this._logger.LogInformation($"{dataTable.Rows.Count} row(s) queried from database: {connection.Database} using Command: {command.CommandText}"); - return JsonConvert.SerializeObject(dataTable); + // Serialize any DateTime objects in UTC format + var jsonSerializerSettings = new JsonSerializerSettings() + { + DateFormatString = ISO_8061_DATETIME_FORMAT + }; + return JsonConvert.SerializeObject(dataTable, jsonSerializerSettings); } } diff --git a/src/packages.lock.json b/src/packages.lock.json index 48b0acef4..220fd768e 100644 --- a/src/packages.lock.json +++ b/src/packages.lock.json @@ -4,34 +4,33 @@ ".NETStandard,Version=v2.0": { "Microsoft.ApplicationInsights": { "type": "Direct", - "requested": "[2.14.0, )", - "resolved": "2.14.0", - "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", + "requested": "[2.17.0, )", + "resolved": "2.17.0", + "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4" + "System.Diagnostics.DiagnosticSource": "5.0.0" } }, "Microsoft.AspNetCore.Http": { "type": "Direct", - "requested": "[2.1.22, )", - "resolved": "2.1.22", - "contentHash": "+Blk++1JWqghbl8+3azQmKhiNZA5wAepL9dY2I6KVmu2Ri07MAcvAVC888qUvO7yd7xgRgZOMfihezKg14O/2A==", + "requested": "[2.2.2, )", + "resolved": "2.2.2", + "contentHash": "BAibpoItxI5puk7YJbIGj95arZueM8B8M5xT1fXBn3hb3L2G3ucrZcYXv1gXdaroLbntUs8qeV8iuBrpjQsrKw==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.1", - "Microsoft.AspNetCore.WebUtilities": "2.1.1", - "Microsoft.Extensions.ObjectPool": "2.1.1", - "Microsoft.Extensions.Options": "2.1.1", - "Microsoft.Net.Http.Headers": "2.1.1" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.Azure.WebJobs": { "type": "Direct", - "requested": "[3.0.31, )", - "resolved": "3.0.31", - "contentHash": "Jn6E7OgT7LkwVB6lCpjXJcoQIvKrbJT+taVLA4CekEpa21pzZv6nQ2sYRSNzPz5ul3FAcYhmrCQgV7v2iopjgA==", + "requested": "[3.0.32, )", + "resolved": "3.0.32", + "contentHash": "uN8GsFqPFHHcSrwwj/+0tGe6F6cOwugqUiePPw7W3TL9YC594+Hw8GBK5S/fcDWXacqvRRGf9nDX8xP94/Yiyw==", "dependencies": { - "Microsoft.Azure.WebJobs.Core": "3.0.31", + "Microsoft.Azure.WebJobs.Core": "3.0.32", "Microsoft.Extensions.Configuration": "2.1.1", "Microsoft.Extensions.Configuration.Abstractions": "2.1.1", "Microsoft.Extensions.Configuration.EnvironmentVariables": "2.1.0", @@ -47,23 +46,26 @@ }, "Microsoft.Data.SqlClient": { "type": "Direct", - "requested": "[3.0.1, )", - "resolved": "3.0.1", - "contentHash": "5Jgcds8yukUeOHvc8S0rGW87rs2uYEM9/YyrYIb/0C+vqzIa2GiqbVPCDVcnApWhs67OSXLTM7lO6jro24v/rA==", - "dependencies": { - "Azure.Identity": "1.3.0", - "Microsoft.Data.SqlClient.SNI.runtime": "3.0.0", - "Microsoft.Identity.Client": "4.22.0", - "Microsoft.IdentityModel.JsonWebTokens": "6.8.0", - "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.8.0", - "Microsoft.Win32.Registry": "4.7.0", + "requested": "[5.0.1, )", + "resolved": "5.0.1", + "contentHash": "uu8dfrsx081cSbEevWuZAvqdmANDGJkbLBL2G3j0LAZxX1Oy8RCVAaC4Lcuak6jNicWP6CWvHqBTIEmQNSxQlw==", + "dependencies": { + "Azure.Identity": "1.6.0", + "Microsoft.Data.SqlClient.SNI.runtime": "5.0.1", + "Microsoft.Identity.Client": "4.45.0", + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.21.0", + "Microsoft.SqlServer.Server": "1.0.0", + "Microsoft.Win32.Registry": "5.0.0", "System.Buffers": "4.5.1", - "System.Configuration.ConfigurationManager": "4.7.0", - "System.Memory": "4.5.4", - "System.Runtime.Caching": "4.7.0", + "System.Configuration.ConfigurationManager": "5.0.0", + "System.IO": "4.3.0", + "System.Resources.ResourceManager": "4.3.0", + "System.Runtime.Caching": "5.0.0", "System.Runtime.Loader": "4.3.0", - "System.Security.Principal.Windows": "4.7.0", - "System.Text.Encoding.CodePages": "4.7.0", + "System.Security.Cryptography.Cng": "5.0.0", + "System.Security.Principal.Windows": "5.0.0", + "System.Text.Encoding.CodePages": "5.0.0", "System.Text.Encodings.Web": "4.7.2" } }, @@ -94,77 +96,77 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[11.0.2, )", - "resolved": "11.0.2", - "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" }, "System.Runtime.Caching": { "type": "Direct", - "requested": "[4.7.0, )", - "resolved": "4.7.0", - "contentHash": "NdvNRjTPxYvIEhXQszT9L9vJhdQoX6AQ0AlhjTU+5NqFQVuacJTfhPVAvtGWNA2OJCqRiR/okBcZgMwI6MqcZg==", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "30D6MkO8WF9jVGWZIP0hmCN8l9BTY4LCsAzLIe4xFSXzs+AjDotR7DpSmj27pFskDURzUvqYYY0ikModgBTxWw==", "dependencies": { - "System.Configuration.ConfigurationManager": "4.7.0" + "System.Configuration.ConfigurationManager": "5.0.0" } }, "Azure.Core": { "type": "Transitive", - "resolved": "1.6.0", - "contentHash": "kI4m2NsODPOrxo0OoKjk6B3ADbdovhDQIEmI4039upjjZKRaewVLx/Uz4DfRa/NtnIRZQPUALe1yvdHWAoRt4w==", + "resolved": "1.24.0", + "contentHash": "+/qI1j2oU1S4/nvxb2k/wDsol00iGf1AyJX5g3epV7eOpQEP/2xcgh/cxgKMeFgn3U2fmgSiBnQZdkV+l5y0Uw==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "1.0.0", - "System.Buffers": "4.5.0", + "Microsoft.Bcl.AsyncInterfaces": "1.1.1", "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.3", + "System.Memory.Data": "1.0.2", "System.Numerics.Vectors": "4.5.0", - "System.Text.Json": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Text.Encodings.Web": "4.7.2", + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Azure.Identity": { "type": "Transitive", - "resolved": "1.3.0", - "contentHash": "l1SYfZKOFBuUFG7C2SWHmJcrQQaiXgBdVCycx4vcZQkC6efDVt7mzZ5pfJAFEJDBUq7mjRQ0RPq9ZDGdSswqMg==", + "resolved": "1.6.0", + "contentHash": "EycyMsb6rD2PK9P0SyibFfEhvWWttdrYhyPF4f41uzdB/44yQlV+2Wehxyg489Rj6gbPvSPgbKq0xsHJBhipZA==", "dependencies": { - "Azure.Core": "1.6.0", - "Microsoft.Identity.Client": "4.22.0", - "Microsoft.Identity.Client.Extensions.Msal": "2.16.5", - "System.Memory": "4.5.3", - "System.Security.Cryptography.ProtectedData": "4.5.0", - "System.Text.Json": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "Azure.Core": "1.24.0", + "Microsoft.Identity.Client": "4.39.0", + "Microsoft.Identity.Client.Extensions.Msal": "2.19.3", + "System.Memory": "4.5.4", + "System.Security.Cryptography.ProtectedData": "4.7.0", + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Microsoft.AspNetCore.Http.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "kQUEVOU4loc8CPSb2WoHFTESqwIa8Ik7ysCBfTwzHAd0moWovc9JQLmhDIHlYLjHbyexqZAlkq/FPRUZqokebw==", + "resolved": "2.2.0", + "contentHash": "Nxs7Z1q3f1STfLYKJSVXCs1iBl+Ya6E8o4Oy1bCxJ/rNI44E/0f6tbsrVqAWfB7jlnJfyaAtIalBVxPKUPQb4Q==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.1", + "Microsoft.AspNetCore.Http.Features": "2.2.0", "System.Text.Encodings.Web": "4.5.0" } }, "Microsoft.AspNetCore.Http.Features": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VklZ7hWgSvHBcDtwYYkdMdI/adlf7ebxTZ9kdzAhX+gUs5jSHE9mZlTamdgf9miSsxc1QjNazHXTDJdVPZKKTw==", + "resolved": "2.2.0", + "contentHash": "ziFz5zH8f33En4dX81LW84I6XrYXKf9jg6aM39cM+LffN9KJahViKZ61dGMSO2gd3e+qe5yBRwsesvyqlZaSMg==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.WebUtilities": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "PGKIZt4+412Z/XPoSjvYu/QIbTxcAQuEFNoA1Pw8a9mgmO0ZhNBmfaNyhgXFf7Rq62kP0tT/2WXpxdcQhkFUPA==", + "resolved": "2.2.0", + "contentHash": "9ErxAAKaDzxXASB/b5uLEkLgUWv1QbeVxyJYEHQwMaxXOeFFVkQxiq8RyfVcifLU7NR0QY0p3acqx4ZpYfhHDg==", "dependencies": { - "Microsoft.Net.Http.Headers": "2.1.1", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Text.Encodings.Web": "4.5.0" } }, "Microsoft.Azure.WebJobs.Core": { "type": "Transitive", - "resolved": "3.0.31", - "contentHash": "iStV0MQ9env8R2F+8cbaynMK4TDkU6bpPVLdOzIs83iriHYMF+uWF6WZWI8ZJahWR37puQWc86u1YCsoIZEocA==", + "resolved": "3.0.32", + "contentHash": "pW5lyF0Tno1cC2VkmBLyv7E3o5ObDdbn3pfpUpKdksJo9ysCdQTpgc0Ib99wPHca6BgvoglicGbDYXuatanMfg==", "dependencies": { "System.ComponentModel.Annotations": "4.4.0", "System.Diagnostics.TraceSource": "4.3.0" @@ -172,10 +174,10 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "1.0.0", - "contentHash": "K63Y4hORbBcKLWH5wnKgzyn7TOfYzevIEwIedQHBIkmkEBA9SCqgvom+XTuE+fAFGvINGkhFItaZ2dvMGdT5iw==", + "resolved": "1.1.1", + "contentHash": "yuvf07qFWFqtK3P/MRkEKLhn5r2UbSpVueRziSqj0yJQIKFwG1pq9mOayK3zE5qZCTs0CbrwL9M6R8VwqyGy2w==", "dependencies": { - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Microsoft.Build.Tasks.Git": { @@ -190,8 +192,8 @@ }, "Microsoft.Data.SqlClient.SNI.runtime": { "type": "Transitive", - "resolved": "3.0.0", - "contentHash": "n1sNyjJgu2pYWKgw3ZPikw3NiRvG4kt7Ya5MK8u77Rgj/1bTFqO/eDF4k5W9H5GXplMZCpKkNbp5kNBICgSB0w==" + "resolved": "5.0.1", + "contentHash": "y0X5MxiNdbITJYoafJ2ruaX6hqO0twpCGR/ipiDOe85JKLU8WL4TuAQfDe5qtt3bND5Je26HnrarLSAMMnVTNg==" }, "Microsoft.Extensions.Configuration": { "type": "Transitive", @@ -254,8 +256,8 @@ }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "MgYpU5cwZohUMKKg3sbPhvGG+eAZ/59E9UwPwlrUkyXU+PGzqwZg9yyQNjhxuAWmoNoFReoemeCku50prYSGzA==" + "resolved": "2.2.0", + "contentHash": "f9hstgjVmr6rmrfGSpfsVOl2irKAgr1QjrSi3FgnS7kulxband50f2brRLwySAQTADPZeTdow0mpSMcoAdadCw==" }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", @@ -329,16 +331,17 @@ }, "Microsoft.Extensions.ObjectPool": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "SErON45qh4ogDp6lr6UvVmFYW0FERihW+IQ+2JyFv1PUyWktcJytFaWH5zarufJvZwhci7Rf1IyGXr9pVEadTw==" + "resolved": "2.2.0", + "contentHash": "gA8H7uQOnM5gb+L0uTNjViHYr+hRDqCdfugheGo/MxQnuHzmhhzCBTIPm19qL1z1Xe0NEMabfcOBGv9QghlZ8g==" }, "Microsoft.Extensions.Options": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "V7lXCU78lAbzaulCGFKojcCyG8RTJicEbiBkPJjFqiqXwndEBBIehdXRMWEVU3UtzQ1yDvphiWUL9th6/4gJ7w==", + "resolved": "2.2.0", + "contentHash": "UpZLNLBpIZ0GTebShui7xXYh6DmBHjWM8NxGxZbdQh/bPZ5e6YswqI+bru6BnEL5eWiOdodsXtEz3FROcgi/qg==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.1", - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.Primitives": "2.2.0", + "System.ComponentModel.Annotations": "4.5.0" } }, "Microsoft.Extensions.Options.ConfigurationExtensions": { @@ -354,8 +357,8 @@ }, "Microsoft.Extensions.Primitives": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "scJ1GZNIxMmjpENh0UZ8XCQ6vzr/LzeF9WvEA51Ix2OQGAs9WPgPu8ABVUdvpKPLuor/t05gm6menJK3PwqOXg==", + "resolved": "2.2.0", + "contentHash": "azyQtqbm4fSaDzZHD/J+V6oWMFaf2tWP4WEGIYePLCMw3+b2RQdj9ybgbQyjCshcitQKQ4lEDOZjmSlTTrHxUg==", "dependencies": { "System.Memory": "4.5.1", "System.Runtime.CompilerServices.Unsafe": "4.5.1" @@ -363,11 +366,11 @@ }, "Microsoft.Identity.Client": { "type": "Transitive", - "resolved": "4.22.0", - "contentHash": "GlamU9rs8cSVIx9WSGv5QKpt66KkE+ImxNa/wNZZUJ3knt3PM98T9sOY8B7NcEfhw7NoxU2/0TSOcmnRSJQgqw==", + "resolved": "4.45.0", + "contentHash": "ircobISCLWbtE5eEoLKU+ldfZ8O41vg4lcy38KRj/znH17jvBiAl8oxcyNp89CsuqE3onxIpn21Ca7riyDDrRw==", "dependencies": { "Microsoft.CSharp": "4.5.0", - "NETStandard.Library": "1.6.1", + "Microsoft.IdentityModel.Abstractions": "6.18.0", "System.ComponentModel.TypeConverter": "4.3.0", "System.Diagnostics.Process": "4.3.0", "System.Dynamic.Runtime": "4.3.0", @@ -375,6 +378,7 @@ "System.Runtime.Serialization.Formatters": "4.3.0", "System.Runtime.Serialization.Json": "4.3.0", "System.Runtime.Serialization.Primitives": "4.3.0", + "System.Security.Claims": "4.3.0", "System.Security.Cryptography.X509Certificates": "4.3.0", "System.Security.SecureString": "4.3.0", "System.Xml.XDocument": "4.3.0", @@ -383,60 +387,68 @@ }, "Microsoft.Identity.Client.Extensions.Msal": { "type": "Transitive", - "resolved": "2.16.5", - "contentHash": "VlGUZEpF8KP/GCfFI59sdE0WA0o9quqwM1YQY0dSp6jpGy5EOBkureaybLfpwCuYUUjQbLkN2p7neUIcQCfbzA==", + "resolved": "2.19.3", + "contentHash": "zVVZjn8aW7W79rC1crioDgdOwaFTQorsSO6RgVlDDjc7MvbEGz071wSNrjVhzR0CdQn6Sefx7Abf1o7vasmrLg==", "dependencies": { - "Microsoft.Identity.Client": "4.22.0", + "Microsoft.Identity.Client": "4.38.0", "System.Security.Cryptography.ProtectedData": "4.5.0" } }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "XeE6LQtD719Qs2IG7HDi1TSw9LIkDbJ33xFiOBoHbApVw/8GpIBCbW+t7RwOjErUDyXZvjhZliwRkkLb8Z1uzg==" + }, "Microsoft.IdentityModel.JsonWebTokens": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "+7JIww64PkMt7NWFxoe4Y/joeF7TAtA/fQ0b2GFGcagzB59sKkTt/sMZWR6aSZht5YC7SdHi3W6yM1yylRGJCQ==", + "resolved": "6.21.0", + "contentHash": "d3h1/BaMeylKTkdP6XwRCxuOoDJZ44V9xaXr6gl5QxmpnZGdoK3bySo3OQN8ehRLJHShb94ElLUvoXyglQtgAw==", "dependencies": { - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "Microsoft.IdentityModel.Logging": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "Rfh/p4MaN4gkmhPxwbu8IjrmoDncGfHHPh1sTnc0AcM/Oc39/fzC9doKNWvUAjzFb8LqA6lgZyblTrIsX/wDXg==" + "resolved": "6.21.0", + "contentHash": "tuEhHIQwvBEhMf8I50hy8FHmRSUkffDFP5EdLsSDV4qRcl2wvOPkQxYqEzWkh+ytW6sbdJGEXElGhmhDfAxAKg==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.21.0" + } }, "Microsoft.IdentityModel.Protocols": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "OJZx5nPdiH+MEkwCkbJrTAUiO/YzLe0VSswNlDxJsJD9bhOIdXHufh650pfm59YH1DNevp3/bXzukKrG57gA1w==", + "resolved": "6.21.0", + "contentHash": "0FqY5cTLQKtHrClzHEI+QxJl8OBT2vUiEQQB7UKk832JDiJJmetzYZ3AdSrPjN/3l3nkhByeWzXnhrX0JbifKg==", "dependencies": { - "Microsoft.IdentityModel.Logging": "6.8.0", - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.Logging": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "Microsoft.IdentityModel.Protocols.OpenIdConnect": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "X/PiV5l3nYYsodtrNMrNQIVlDmHpjQQ5w48E+o/D5H4es2+4niEyQf3l03chvZGWNzBRhfSstaXr25/Ye4AeYw==", + "resolved": "6.21.0", + "contentHash": "vtSKL7n6EnAsLyxmiviusm6LKrblT2ndnNqN6rvVq6iIHAnPCK9E2DkDx6h1Jrpy1cvbp40r0cnTg23nhEAGTA==", "dependencies": { - "Microsoft.IdentityModel.Protocols": "6.8.0", - "System.IdentityModel.Tokens.Jwt": "6.8.0" + "Microsoft.IdentityModel.Protocols": "6.21.0", + "System.IdentityModel.Tokens.Jwt": "6.21.0" } }, "Microsoft.IdentityModel.Tokens": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "gTqzsGcmD13HgtNePPcuVHZ/NXWmyV+InJgalW/FhWpII1D7V1k0obIseGlWMeA4G+tZfeGMfXr0klnWbMR/mQ==", + "resolved": "6.21.0", + "contentHash": "AAEHZvZyb597a+QJSmtxH3n2P1nIJGpZ4Q89GTenknRx6T6zyfzf592yW/jA5e8EHN4tNMjjXHQaYWEq5+L05w==", "dependencies": { "Microsoft.CSharp": "4.5.0", - "Microsoft.IdentityModel.Logging": "6.8.0", + "Microsoft.IdentityModel.Logging": "6.21.0", "System.Security.Cryptography.Cng": "4.5.0" } }, "Microsoft.Net.Http.Headers": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "lPNIphl8b2EuhOE9dMH6EZDmu7pS882O+HMi5BJNsigxHaWlBrYxZHFZgE18cyaPp6SSZcTkKkuzfjV/RRQKlA==", + "resolved": "2.2.0", + "contentHash": "iZNkjYqlo8sIOI0bQfpsSoMTmB/kyvmV2h225ihyZT33aTp48ZpF6qYnXxzSXmHt8DpBAwBTX+1s1UFLbYfZKg==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1", + "Microsoft.Extensions.Primitives": "2.2.0", "System.Buffers": "4.5.0" } }, @@ -455,6 +467,11 @@ "resolved": "1.1.1", "contentHash": "WMcGpWKrmJmzrNeuaEb23bEMnbtR/vLmvZtkAP5qWu7vQsY59GqfRJd65sFpBszbd2k/bQ8cs8eWawQKAabkVg==" }, + "Microsoft.SqlServer.Server": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "N4KeF3cpcm1PUHym1RmakkzfkEv3GRMyofVv40uXsQhCQeglr2OHNcUk2WOG51AKpGO8ynGpo9M/kFXSzghwug==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -467,13 +484,13 @@ }, "Microsoft.Win32.Registry": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "KSrRMb5vNi0CWSGG1++id2ZOs/1QhRqROt+qgbEAdQuGjGrFcl4AOl4/exGPUYz2wUnU42nvJqon1T3U0kPXLA==", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", "dependencies": { - "System.Buffers": "4.5.0", - "System.Memory": "4.5.3", - "System.Security.AccessControl": "4.7.0", - "System.Security.Principal.Windows": "4.7.0" + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4", + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" } }, "runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl": { @@ -643,8 +660,8 @@ }, "System.ComponentModel.Annotations": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "29K3DQ+IGU7LBaMjTo7SI7T7X/tsMtLvz1p56LJ556Iu0Dw3pKZw5g8yCYCWMRxrOF0Hr0FU0FwW0o42y2sb3A==" + "resolved": "4.5.0", + "contentHash": "UxYQ3FGUOtzJ7LfSdnYSFd7+oEv6M8NgUatatIN2HxNtDdlcvFAf+VIq4Of9cDMJEJC0aSRv/x898RYhB4Yppg==" }, "System.ComponentModel.Primitives": { "type": "Transitive", @@ -680,11 +697,11 @@ }, "System.Configuration.ConfigurationManager": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "/anOTeSZCNNI2zDilogWrZ8pNqCmYbzGNexUnNhjW8k0sHqEZ2nHJBp147jBV3hGYswu5lINpNg1vxR7bnqvVA==", + "resolved": "5.0.0", + "contentHash": "aM7cbfEfVNlEEOj3DsZP+2g9NRwbkyiAv2isQEzw7pnkDg9ekCU2m1cdJLM02Uq691OaCS91tooaxcEn8d0q5w==", "dependencies": { - "System.Security.Cryptography.ProtectedData": "4.7.0", - "System.Security.Permissions": "4.7.0" + "System.Security.Cryptography.ProtectedData": "5.0.0", + "System.Security.Permissions": "5.0.0" } }, "System.Diagnostics.Debug": { @@ -699,10 +716,11 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "4.6.0", - "contentHash": "mbBgoR0rRfl2uimsZ2avZY8g7Xnh1Mza0rJZLPcxqiMWlkGukjmRkuMJ/er+AhQuiRIh80CR/Hpeztr80seV5g==", + "resolved": "5.0.0", + "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==", "dependencies": { - "System.Memory": "4.5.3" + "System.Memory": "4.5.4", + "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, "System.Diagnostics.Process": { @@ -826,11 +844,11 @@ }, "System.IdentityModel.Tokens.Jwt": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "5tBCjAub2Bhd5qmcd0WhR5s354e4oLYa//kOWrkX+6/7ZbDDJjMTfwLSOiZ/MMpWdE4DWPLOfTLOq/juj9CKzA==", + "resolved": "6.21.0", + "contentHash": "JRD8AuypBE+2zYxT3dMJomQVsPYsCqlyZhWel3J1d5nzQokSRyTueF+Q4ID3Jcu6zSZKuzOdJ1MLTkbQsDqcvQ==", "dependencies": { - "Microsoft.IdentityModel.JsonWebTokens": "6.8.0", - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "System.IO": { @@ -916,9 +934,10 @@ }, "System.Memory.Data": { "type": "Transitive", - "resolved": "1.0.1", - "contentHash": "ujvrOjcni2QQbr6hG2AkUTWLb/xplrx0mt6HrdHFCzzGky2d5J6YD60TKAEf8SBk33cfSzTvFmXewAVaPY/dZg==", + "resolved": "1.0.2", + "contentHash": "JGkzeqgBsiZwKJZ1IxPNsDFZDhUvuEdX8L8BDC8N3KOj+6zMcNU28CNN59TpZE/VJYy9cP+5M+sbxtWJx3/xtw==", "dependencies": { + "System.Text.Encodings.Web": "4.7.2", "System.Text.Json": "4.6.0" } }, @@ -1078,8 +1097,8 @@ }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "IpU1lcHz8/09yDr9N+Juc7SCgNUz+RohkCQI+KsWKR67XxpFr8Z6c8t1iENCXZuRuNCc4HBwme/MDHNVCwyAKg==" + "resolved": "5.0.0", + "contentHash": "ZD9TMpsmYJLrxbbmdvhwt9YEgG5WntEnZ/d1eH8JBX9LBp+Ju8BSBhUGbZMNVHHomWo2KVImJhTDl2hIgw/6MA==" }, "System.Runtime.Extensions": { "type": "Transitive", @@ -1168,10 +1187,24 @@ }, "System.Security.AccessControl": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "JECvTt5aFF3WT3gHpfofL2MNNP6v84sxtXxpqhLBCcDRzqsPBmHhQ6shv4DwwN2tRlzsUxtb3G9M3763rbXKDg==", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", "dependencies": { - "System.Security.Principal.Windows": "4.7.0" + "System.Security.Principal.Windows": "5.0.0" + } + }, + "System.Security.Claims": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "P/+BR/2lnc4PNDHt/TPBAWHVMLMRHsyYZbU1NphW4HIWzCggz8mJbTQQ3MKljFE7LS3WagmVFuBgoLcFzYXlkA==", + "dependencies": { + "System.Collections": "4.3.0", + "System.Globalization": "4.3.0", + "System.IO": "4.3.0", + "System.Resources.ResourceManager": "4.3.0", + "System.Runtime": "4.3.0", + "System.Runtime.Extensions": "4.3.0", + "System.Security.Principal": "4.3.0" } }, "System.Security.Cryptography.Algorithms": { @@ -1197,8 +1230,8 @@ }, "System.Security.Cryptography.Cng": { "type": "Transitive", - "resolved": "4.5.0", - "contentHash": "WG3r7EyjUe9CMPFSs6bty5doUqT+q9pbI80hlNzo2SkPkZ4VTuZkGWjpp77JB8+uaL4DFPRdBsAY+DX3dBK92A==" + "resolved": "5.0.0", + "contentHash": "jIMXsKn94T9JY7PvPq/tMfqa6GAaHpElRDpmG+SuL+D3+sTw2M8VhnibKnN8Tq+4JqbPJ/f+BwtLeDMEnzAvRg==" }, "System.Security.Cryptography.Csp": { "type": "Transitive", @@ -1275,10 +1308,10 @@ }, "System.Security.Cryptography.ProtectedData": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "ehYW0m9ptxpGWvE4zgqongBVWpSDU/JCFD4K7krxkQwSz/sFQjEXCUqpvencjy6DYDbn7Ig09R8GFffu8TtneQ==", + "resolved": "5.0.0", + "contentHash": "HGxMSAFAPLNoxBvSfW08vHde0F9uh7BjASwu6JF9JnXuEPhCY3YUqURn0+bQV/4UWeaqymmrHWV+Aw9riQCtCA==", "dependencies": { - "System.Memory": "4.5.3" + "System.Memory": "4.5.4" } }, "System.Security.Cryptography.X509Certificates": { @@ -1315,16 +1348,24 @@ }, "System.Security.Permissions": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "dkOV6YYVBnYRa15/yv004eCGRBVADXw8qRbbNiCn/XpdJSUXkkUeIvdvFHkvnko4CdKMqG8yRHC4ox83LSlMsQ==", + "resolved": "5.0.0", + "contentHash": "uE8juAhEkp7KDBCdjDIE3H9R1HJuEHqeqX8nLX9gmYKWwsqk3T5qZlPx8qle5DPKimC/Fy3AFTdV7HamgCh9qQ==", + "dependencies": { + "System.Security.AccessControl": "5.0.0" + } + }, + "System.Security.Principal": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "I1tkfQlAoMM2URscUtpcRo/hX0jinXx6a/KUtEQoz3owaYwl3qwsO8cbzYVVnjxrzxjHo3nJC+62uolgeGIS9A==", "dependencies": { - "System.Security.AccessControl": "4.7.0" + "System.Runtime": "4.3.0" } }, "System.Security.Principal.Windows": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "ojD0PX0XhneCsUbAZVKdb7h/70vyYMDYs85lwEI+LngEONe/17A0cFaRFqZU+sOEidcVswYWikYOQ9PPfjlbtQ==" + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" }, "System.Security.SecureString": { "type": "Transitive", @@ -1353,10 +1394,10 @@ }, "System.Text.Encoding.CodePages": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "aeu4FlaUTemuT1qOd1MyU4T516QR4Fy+9yDbwWMPHOHy7U8FD6SgTzdZFO7gHcfAPHtECqInbwklVvUK4RHcNg==", + "resolved": "5.0.0", + "contentHash": "NyscU59xX6Uo91qvhOs2Ccho3AR2TnZPomo1Z0K6YpyztBPM/A5VbkzOO19sy3A3i1TtEnTxA7bCe3Us+r5MWg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.7.0" + "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, "System.Text.Encoding.Extensions": { @@ -1381,16 +1422,16 @@ }, "System.Text.Json": { "type": "Transitive", - "resolved": "4.6.0", - "contentHash": "4F8Xe+JIkVoDJ8hDAZ7HqLkjctN/6WItJIzQaifBwClC7wmoLSda/Sv2i6i1kycqDb3hWF4JCVbpAweyOKHEUA==", + "resolved": "4.7.2", + "contentHash": "TcMd95wcrubm9nHvJEQs70rC0H/8omiSGGpU4FQ/ZA1URIqD4pjmFJh2Mfv1yH1eHgJDWTi2hMDXwTET+zOOyg==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "1.0.0", - "System.Buffers": "4.5.0", - "System.Memory": "4.5.3", + "Microsoft.Bcl.AsyncInterfaces": "1.1.0", + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4", "System.Numerics.Vectors": "4.5.0", - "System.Runtime.CompilerServices.Unsafe": "4.6.0", - "System.Text.Encodings.Web": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Runtime.CompilerServices.Unsafe": "4.7.1", + "System.Text.Encodings.Web": "4.7.1", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "System.Text.RegularExpressions": { @@ -1432,10 +1473,10 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.2", - "contentHash": "BG/TNxDFv0svAzx8OiMXDlsHfGw623BZ8tCXw4YLhDFDvDhNUEV58jKYMGRnkbJNm7c3JNNJDiN7JBMzxRBR2w==", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "4.5.2" + "System.Runtime.CompilerServices.Unsafe": "4.5.3" } }, "System.Threading.Thread": { diff --git a/test/Common/SupportedLanguagesTestAttribute.cs b/test/Common/SupportedLanguagesTestAttribute.cs index 435bad3b5..c2cd73791 100644 --- a/test/Common/SupportedLanguagesTestAttribute.cs +++ b/test/Common/SupportedLanguagesTestAttribute.cs @@ -14,7 +14,7 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common /// public class SqlInlineDataAttribute : DataAttribute { - private readonly List testData = new List(); + private readonly List testData = new(); /// /// Adds a language parameter to the test data which will contain the language diff --git a/test/Common/TestData.cs b/test/Common/TestData.cs index 0596399e3..39667b095 100644 --- a/test/Common/TestData.cs +++ b/test/Common/TestData.cs @@ -17,7 +17,7 @@ public class TestData public override bool Equals(object obj) { - if (!(obj is TestData otherData)) + if (obj is not TestData otherData) { return false; } diff --git a/test/Common/TestUtils.cs b/test/Common/TestUtils.cs index 8ba874b9f..387418a81 100644 --- a/test/Common/TestUtils.cs +++ b/test/Common/TestUtils.cs @@ -29,7 +29,7 @@ public static string GetUniqueDBName(string namePrefix) namePrefix, safeMachineName, AppDomain.CurrentDomain.Id, - Process.GetCurrentProcess().Id, + Environment.ProcessId, Interlocked.Increment(ref ThreadId)); } diff --git a/test/GlobalSuppressions.cs b/test/GlobalSuppressions.cs index b0fa905ab..54b3acf93 100644 --- a/test/GlobalSuppressions.cs +++ b/test/GlobalSuppressions.cs @@ -17,5 +17,5 @@ [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.ReservedPrimaryKeyColumnNamesTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.TableNotPresentTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] [assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration.UnsupportedColumnTypesTrigger.Run(System.Collections.Generic.IReadOnlyList{Microsoft.Azure.WebJobs.Extensions.Sql.SqlChange{Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common.Product})")] -[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples.GetProductsColumnTypesSerializationDifferentCulture.Run(Microsoft.AspNetCore.Http.HttpRequest,System.Collections.Generic.IAsyncEnumerable{Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common.ProductColumnTypes},Microsoft.Extensions.Logging.ILogger)~System.Threading.Tasks.Task{Microsoft.AspNetCore.Mvc.IActionResult}")] -[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples.GetProductsColumnTypesSerialization.Run(Microsoft.AspNetCore.Http.HttpRequest,System.Collections.Generic.IAsyncEnumerable{Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common.ProductColumnTypes},Microsoft.Extensions.Logging.ILogger)~System.Threading.Tasks.Task{Microsoft.AspNetCore.Mvc.IActionResult}")] +[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples.GetProductsColumnTypesSerializationAsyncEnumerable.Run(Microsoft.AspNetCore.Http.HttpRequest,System.Collections.Generic.IAsyncEnumerable{Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common.ProductColumnTypes},Microsoft.Extensions.Logging.ILogger)~System.Threading.Tasks.Task{Microsoft.AspNetCore.Mvc.IActionResult}")] +[assembly: SuppressMessage("Style", "IDE0060:Remove unused parameter", Justification = "Unused parameter is required by functions binding", Scope = "member", Target = "~M:Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples.GetProductsColumnTypesSerialization.Run(Microsoft.AspNetCore.Http.HttpRequest,System.Collections.Generic.IEnumerable{Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common.ProductColumnTypes},Microsoft.Extensions.Logging.ILogger)~Microsoft.AspNetCore.Mvc.IActionResult")] diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index 17acd4bfb..5d6a80a8e 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -104,7 +104,8 @@ private void SetupDatabase() { DataSource = testServer, InitialCatalog = "master", - Pooling = false + Pooling = false, + Encrypt = SqlConnectionEncryptOption.Optional }; // Either use integrated auth or SQL login depending if SA_PASSWORD is set diff --git a/test/Integration/SqlInputBindingIntegrationTests.cs b/test/Integration/SqlInputBindingIntegrationTests.cs index 7b63908a1..7822b1ee9 100644 --- a/test/Integration/SqlInputBindingIntegrationTests.cs +++ b/test/Integration/SqlInputBindingIntegrationTests.cs @@ -132,24 +132,28 @@ public async void GetProductNamesViewTest(SupportedLanguages lang) } /// - /// Verifies that serializing an item with various data types works when the language is - /// set to a non-enUS language. + /// Verifies that serializing an item with various data types and different cultures works when using IAsyncEnumerable /// [Theory] - [SqlInlineData()] - [UnsupportedLanguages(SupportedLanguages.JavaScript)] // Javascript doesn't have the concept of a runtime language used during serialization - public async void GetProductsColumnTypesSerializationDifferentCultureTest(SupportedLanguages lang) + [SqlInlineData("en-US")] + [SqlInlineData("it-IT")] + [UnsupportedLanguages(SupportedLanguages.JavaScript)] // IAsyncEnumerable is only available in C# + public async void GetProductsColumnTypesSerializationAsyncEnumerableTest(string culture, SupportedLanguages lang) { - this.StartFunctionHost(nameof(GetProductsColumnTypesSerializationDifferentCulture), lang, true); + this.StartFunctionHost(nameof(GetProductsColumnTypesSerializationAsyncEnumerable), lang, true); + string datetime = "2022-10-20 12:39:13.123"; this.ExecuteNonQuery("INSERT INTO [dbo].[ProductsColumnTypes] VALUES (" + "999, " + // ProductId - "GETDATE(), " + // Datetime field - "GETDATE())"); // Datetime2 field + $"CONVERT(DATETIME, '{datetime}'), " + // Datetime field + $"CONVERT(DATETIME2, '{datetime}'))"); // Datetime2 field - await this.SendInputRequest("getproducts-columntypesserializationdifferentculture"); + HttpResponseMessage response = await this.SendInputRequest("getproducts-columntypesserializationasyncenumerable", $"?culture={culture}"); + // We expect the datetime and datetime2 fields to be returned in UTC format + string expectedResponse = "[{\"productId\":999,\"datetime\":\"2022-10-20T12:39:13.123Z\",\"datetime2\":\"2022-10-20T12:39:13.123Z\"}]"; + string actualResponse = await response.Content.ReadAsStringAsync(); - // If we get here the test has succeeded - it'll throw an exception if serialization fails + Assert.Equal(expectedResponse, TestUtils.CleanJsonString(actualResponse), StringComparer.OrdinalIgnoreCase); } /// @@ -161,14 +165,18 @@ public async void GetProductsColumnTypesSerializationTest(SupportedLanguages lan { this.StartFunctionHost(nameof(GetProductsColumnTypesSerialization), lang, true); + string datetime = "2022-10-20 12:39:13.123"; this.ExecuteNonQuery("INSERT INTO [dbo].[ProductsColumnTypes] VALUES (" + "999, " + // ProductId - "GETDATE(), " + // Datetime field - "GETDATE())"); // Datetime2 field + $"CONVERT(DATETIME, '{datetime}'), " + // Datetime field + $"CONVERT(DATETIME2, '{datetime}'))"); // Datetime2 field - await this.SendInputRequest("getproducts-columntypesserialization"); + HttpResponseMessage response = await this.SendInputRequest("getproducts-columntypesserialization"); + // We expect the datetime and datetime2 fields to be returned in UTC format + string expectedResponse = "[{\"ProductId\":999,\"Datetime\":\"2022-10-20T12:39:13.123Z\",\"Datetime2\":\"2022-10-20T12:39:13.123Z\"}]"; + string actualResponse = await response.Content.ReadAsStringAsync(); - // If we get here the test has succeeded - it'll throw an exception if serialization fails + Assert.Equal(expectedResponse, TestUtils.CleanJsonString(actualResponse), StringComparer.OrdinalIgnoreCase); } } } diff --git a/test/Integration/test-csharp/GetProductColumnTypesSerialization.cs b/test/Integration/test-csharp/GetProductColumnTypesSerialization.cs index 836c305ca..06de59dc6 100644 --- a/test/Integration/test-csharp/GetProductColumnTypesSerialization.cs +++ b/test/Integration/test-csharp/GetProductColumnTypesSerialization.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Text.Json; -using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs.Extensions.Http; @@ -16,24 +15,23 @@ public static class GetProductsColumnTypesSerialization { /// /// This function verifies that serializing an item with various data types - /// works as expected. - /// Note this uses IAsyncEnumerable because IEnumerable serializes the entire table directly, - /// instead of each item one by one (which is where issues can occur) + /// works as expected when using IEnumerable. /// [FunctionName(nameof(GetProductsColumnTypesSerialization))] - public static async Task Run( + public static IActionResult Run( [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "getproducts-columntypesserialization")] HttpRequest req, [Sql("SELECT * FROM [dbo].[ProductsColumnTypes]", CommandType = System.Data.CommandType.Text, ConnectionStringSetting = "SqlConnectionString")] - IAsyncEnumerable products, + IEnumerable products, ILogger log) { - await foreach (ProductColumnTypes item in products) + foreach (ProductColumnTypes item in products) { log.LogInformation(JsonSerializer.Serialize(item)); } + return new OkObjectResult(products); } } diff --git a/test/Integration/test-csharp/GetProductColumnTypesSerializationDifferentCulture.cs b/test/Integration/test-csharp/GetProductColumnTypesSerializationAsyncEnumerable.cs similarity index 63% rename from test/Integration/test-csharp/GetProductColumnTypesSerializationDifferentCulture.cs rename to test/Integration/test-csharp/GetProductColumnTypesSerializationAsyncEnumerable.cs index 9417ba46a..d8ea55a4d 100644 --- a/test/Integration/test-csharp/GetProductColumnTypesSerializationDifferentCulture.cs +++ b/test/Integration/test-csharp/GetProductColumnTypesSerializationAsyncEnumerable.cs @@ -13,17 +13,15 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Samples.InputBindingSamples { - public static class GetProductsColumnTypesSerializationDifferentCulture + public static class GetProductsColumnTypesSerializationAsyncEnumerable { /// /// This function verifies that serializing an item with various data types - /// works when the language is set to a non-enUS language. - /// Note this uses IAsyncEnumerable because IEnumerable serializes the entire table directly, - /// instead of each item one by one (which is where issues can occur) + /// and different languages works when using IAsyncEnumerable. /// - [FunctionName(nameof(GetProductsColumnTypesSerializationDifferentCulture))] + [FunctionName(nameof(GetProductsColumnTypesSerializationAsyncEnumerable))] public static async Task Run( - [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "getproducts-columntypesserializationdifferentculture")] + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "getproducts-columntypesserializationasyncenumerable")] HttpRequest req, [Sql("SELECT * FROM [dbo].[ProductsColumnTypes]", CommandType = System.Data.CommandType.Text, @@ -31,12 +29,21 @@ public static async Task Run( IAsyncEnumerable products, ILogger log) { - CultureInfo.CurrentCulture = new CultureInfo("it-IT", false); + // Test different cultures to ensure that serialization/deserialization works correctly for all types. + // We expect the datetime types to be serialized in UTC format. + string language = req.Query["culture"]; + if (!string.IsNullOrEmpty(language)) + { + CultureInfo.CurrentCulture = new CultureInfo(language); + } + + var productsList = new List(); await foreach (ProductColumnTypes item in products) { log.LogInformation(JsonSerializer.Serialize(item)); + productsList.Add(item); } - return new OkObjectResult(products); + return new OkObjectResult(productsList); } } } diff --git a/test/Unit/SqlInputBindingTests.cs b/test/Unit/SqlInputBindingTests.cs index 8da337d07..3d54c6a18 100644 --- a/test/Unit/SqlInputBindingTests.cs +++ b/test/Unit/SqlInputBindingTests.cs @@ -19,11 +19,11 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Unit { public class SqlInputBindingTests { - private static readonly Mock config = new Mock(); - private static readonly Mock hostIdProvider = new Mock(); - private static readonly Mock loggerFactory = new Mock(); - private static readonly Mock logger = new Mock(); - private static readonly SqlConnection connection = new SqlConnection(); + private static readonly Mock config = new(); + private static readonly Mock hostIdProvider = new(); + private static readonly Mock loggerFactory = new(); + private static readonly Mock logger = new(); + private static readonly SqlConnection connection = new(); [Fact] public void TestNullConfiguration() @@ -69,7 +69,7 @@ public void TestNullArgumentsSqlAsyncEnumerableConstructor() } /// - /// SqlAsyncEnumerable should throw InvalidOperationExcepion when invoked with an invalid connection + /// SqlAsyncEnumerable should throw InvalidOperationExcepion when invoked with an invalid connection /// string setting and It should fail here since we're passing an empty connection string. /// [Fact] @@ -272,7 +272,7 @@ public async void TestMalformedDeserialization() var converter = new Mock>(config.Object, logger.Object); // SQL data is missing a field - string json = "[{ \"ID\":1,\"Name\":\"Broom\",\"Timestamp\":\"2019-11-22T06:32:15\"}]"; + string json = /*lang=json,strict*/ "[{ \"ID\":1,\"Name\":\"Broom\",\"Timestamp\":\"2019-11-22T06:32:15\"}]"; converter.Setup(_ => _.BuildItemFromAttributeAsync(arg, ConvertType.IEnumerable)).ReturnsAsync(json); var list = new List(); var data = new TestData @@ -287,7 +287,7 @@ public async void TestMalformedDeserialization() Assert.True(enActual.ToList().SequenceEqual(list)); // SQL data's columns are named differently than the POCO's fields - json = "[{ \"ID\":1,\"Product Name\":\"Broom\",\"Price\":32.5,\"Timessstamp\":\"2019-11-22T06:32:15\"}]"; + json = /*lang=json,strict*/ "[{ \"ID\":1,\"Product Name\":\"Broom\",\"Price\":32.5,\"Timessstamp\":\"2019-11-22T06:32:15\"}]"; converter.Setup(_ => _.BuildItemFromAttributeAsync(arg, ConvertType.IEnumerable)).ReturnsAsync(json); list = new List(); data = new TestData @@ -301,7 +301,7 @@ public async void TestMalformedDeserialization() Assert.True(enActual.ToList().SequenceEqual(list)); // Confirm that the JSON fields are case-insensitive (technically malformed string, but still works) - json = "[{ \"id\":1,\"nAme\":\"Broom\",\"coSt\":32.5,\"TimEStamp\":\"2019-11-22T06:32:15\"}]"; + json = /*lang=json,strict*/ "[{ \"id\":1,\"nAme\":\"Broom\",\"coSt\":32.5,\"TimEStamp\":\"2019-11-22T06:32:15\"}]"; converter.Setup(_ => _.BuildItemFromAttributeAsync(arg, ConvertType.IEnumerable)).ReturnsAsync(json); list = new List(); data = new TestData diff --git a/test/Unit/SqlOutputBindingTests.cs b/test/Unit/SqlOutputBindingTests.cs index 60b9ff4ca..4b2334cfb 100644 --- a/test/Unit/SqlOutputBindingTests.cs +++ b/test/Unit/SqlOutputBindingTests.cs @@ -13,8 +13,8 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Unit { public class SqlOutputBindingTests { - private static readonly Mock config = new Mock(); - private static readonly Mock logger = new Mock(); + private static readonly Mock config = new(); + private static readonly Mock logger = new(); [Fact] public void TestNullCollectorConstructorArguments() diff --git a/test/packages.lock.json b/test/packages.lock.json index 8a77ec072..898fc594c 100644 --- a/test/packages.lock.json +++ b/test/packages.lock.json @@ -1,32 +1,32 @@ { "version": 2, "dependencies": { - ".NETCoreApp,Version=v3.1": { + "net6.0": { "Microsoft.AspNetCore.Http": { "type": "Direct", - "requested": "[2.1.22, )", - "resolved": "2.1.22", - "contentHash": "+Blk++1JWqghbl8+3azQmKhiNZA5wAepL9dY2I6KVmu2Ri07MAcvAVC888qUvO7yd7xgRgZOMfihezKg14O/2A==", + "requested": "[2.2.2, )", + "resolved": "2.2.2", + "contentHash": "BAibpoItxI5puk7YJbIGj95arZueM8B8M5xT1fXBn3hb3L2G3ucrZcYXv1gXdaroLbntUs8qeV8iuBrpjQsrKw==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.1", - "Microsoft.AspNetCore.WebUtilities": "2.1.1", - "Microsoft.Extensions.ObjectPool": "2.1.1", - "Microsoft.Extensions.Options": "2.1.1", - "Microsoft.Net.Http.Headers": "2.1.1" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.NET.Sdk.Functions": { "type": "Direct", - "requested": "[3.1.1, )", - "resolved": "3.1.1", - "contentHash": "sPPLAjDYroeuIDKwff5B1XrwvGzJm9K9GabXurmfpaXa3M4POy7ngLcG5mm+2pwjTA7e870pIjt1N2DqVQS4yA==", + "requested": "[4.1.3, )", + "resolved": "4.1.3", + "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", - "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "1.2.2", - "Newtonsoft.Json": "11.0.2" + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", + "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", + "Newtonsoft.Json": "13.0.1" } }, "Microsoft.NET.Test.Sdk": { @@ -51,9 +51,9 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[11.0.2, )", - "resolved": "11.0.2", - "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" }, "xunit": { "type": "Direct", @@ -77,32 +77,30 @@ }, "Azure.Core": { "type": "Transitive", - "resolved": "1.19.0", - "contentHash": "lcDjG635DPE4fU5tqSueVMmzrx0QrIfPuY0+y6evHN5GanQ0GB+/4nuMHMmoNPwEow6OUPkJu4cZQxfHJQXPdA==", + "resolved": "1.24.0", + "contentHash": "+/qI1j2oU1S4/nvxb2k/wDsol00iGf1AyJX5g3epV7eOpQEP/2xcgh/cxgKMeFgn3U2fmgSiBnQZdkV+l5y0Uw==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "1.0.0", - "System.Buffers": "4.5.1", + "Microsoft.Bcl.AsyncInterfaces": "1.1.1", "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4", "System.Memory.Data": "1.0.2", "System.Numerics.Vectors": "4.5.0", "System.Text.Encodings.Web": "4.7.2", - "System.Text.Json": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Azure.Identity": { "type": "Transitive", - "resolved": "1.4.0", - "contentHash": "vvjdoDQb9WQyLkD1Uo5KFbwlW7xIsDMihz3yofskym2SimXswbSXuK7QSR1oHnBLBRMdamnVHLpSKQZhJUDejg==", + "resolved": "1.6.0", + "contentHash": "EycyMsb6rD2PK9P0SyibFfEhvWWttdrYhyPF4f41uzdB/44yQlV+2Wehxyg489Rj6gbPvSPgbKq0xsHJBhipZA==", "dependencies": { - "Azure.Core": "1.14.0", - "Microsoft.Identity.Client": "4.30.1", - "Microsoft.Identity.Client.Extensions.Msal": "2.18.4", + "Azure.Core": "1.24.0", + "Microsoft.Identity.Client": "4.39.0", + "Microsoft.Identity.Client.Extensions.Msal": "2.19.3", "System.Memory": "4.5.4", - "System.Security.Cryptography.ProtectedData": "4.5.0", - "System.Text.Json": "4.6.0", - "System.Threading.Tasks.Extensions": "4.5.2" + "System.Security.Cryptography.ProtectedData": "4.7.0", + "System.Text.Json": "4.7.2", + "System.Threading.Tasks.Extensions": "4.5.4" } }, "Azure.Storage.Blobs": { @@ -151,8 +149,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.4", - "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", + "resolved": "5.2.8", + "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -160,93 +158,93 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", + "resolved": "2.2.0", + "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", + "resolved": "2.2.0", + "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", + "resolved": "2.2.0", + "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", + "resolved": "2.2.0", + "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Authorization": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Authorization": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", + "resolved": "2.2.0", + "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", + "resolved": "2.2.0", + "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Features": "2.2.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "kQUEVOU4loc8CPSb2WoHFTESqwIa8Ik7ysCBfTwzHAd0moWovc9JQLmhDIHlYLjHbyexqZAlkq/FPRUZqokebw==", + "resolved": "2.2.0", + "contentHash": "Nxs7Z1q3f1STfLYKJSVXCs1iBl+Ya6E8o4Oy1bCxJ/rNI44E/0f6tbsrVqAWfB7jlnJfyaAtIalBVxPKUPQb4Q==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.1", + "Microsoft.AspNetCore.Http.Features": "2.2.0", "System.Text.Encodings.Web": "4.5.0" } }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", + "resolved": "2.2.0", + "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Buffers": "4.5.0" } }, "Microsoft.AspNetCore.Http.Features": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VklZ7hWgSvHBcDtwYYkdMdI/adlf7ebxTZ9kdzAhX+gUs5jSHE9mZlTamdgf9miSsxc1QjNazHXTDJdVPZKKTw==", + "resolved": "2.2.0", + "contentHash": "ziFz5zH8f33En4dX81LW84I6XrYXKf9jg6aM39cM+LffN9KJahViKZ61dGMSO2gd3e+qe5yBRwsesvyqlZaSMg==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", + "resolved": "2.2.0", + "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -254,88 +252,89 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", + "resolved": "2.2.0", + "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.1.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Extensions.DependencyInjection": "2.1.0", + "resolved": "2.2.0", + "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.2.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection": "2.2.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.0" + "System.Threading.Tasks.Extensions": "4.5.1" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", + "resolved": "2.2.0", + "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.1.0", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0" + "Microsoft.AspNetCore.JsonPatch": "2.2.0", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", + "resolved": "2.2.0", + "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.WebUtilities": "2.1.0" + "Microsoft.AspNet.WebApi.Client": "5.2.6", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", + "resolved": "2.2.0", + "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", + "resolved": "2.2.2", + "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.ObjectPool": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", + "resolved": "2.2.0", + "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.WebUtilities": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "PGKIZt4+412Z/XPoSjvYu/QIbTxcAQuEFNoA1Pw8a9mgmO0ZhNBmfaNyhgXFf7Rq62kP0tT/2WXpxdcQhkFUPA==", + "resolved": "2.2.0", + "contentHash": "9ErxAAKaDzxXASB/b5uLEkLgUWv1QbeVxyJYEHQwMaxXOeFFVkQxiq8RyfVcifLU7NR0QY0p3acqx4ZpYfhHDg==", "dependencies": { - "Microsoft.Net.Http.Headers": "2.1.1", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Text.Encodings.Web": "4.5.0" } }, @@ -346,8 +345,8 @@ }, "Microsoft.Azure.WebJobs.Core": { "type": "Transitive", - "resolved": "3.0.31", - "contentHash": "iStV0MQ9env8R2F+8cbaynMK4TDkU6bpPVLdOzIs83iriHYMF+uWF6WZWI8ZJahWR37puQWc86u1YCsoIZEocA==", + "resolved": "3.0.32", + "contentHash": "pW5lyF0Tno1cC2VkmBLyv7E3o5ObDdbn3pfpUpKdksJo9ysCdQTpgc0Ib99wPHca6BgvoglicGbDYXuatanMfg==", "dependencies": { "System.ComponentModel.Annotations": "4.4.0", "System.Diagnostics.TraceSource": "4.3.0" @@ -365,15 +364,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.0.2", - "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", + "resolved": "3.2.0", + "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Azure.WebJobs": "3.0.2" + "Microsoft.AspNet.WebApi.Client": "5.2.8", + "Microsoft.AspNetCore.Http": "2.2.2", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.2", + "Microsoft.Azure.WebJobs": "3.0.32" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -408,8 +407,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "1.0.0", - "contentHash": "K63Y4hORbBcKLWH5wnKgzyn7TOfYzevIEwIedQHBIkmkEBA9SCqgvom+XTuE+fAFGvINGkhFItaZ2dvMGdT5iw==" + "resolved": "1.1.1", + "contentHash": "yuvf07qFWFqtK3P/MRkEKLhn5r2UbSpVueRziSqj0yJQIKFwG1pq9mOayK3zE5qZCTs0CbrwL9M6R8VwqyGy2w==" }, "Microsoft.CodeCoverage": { "type": "Transitive", @@ -423,8 +422,8 @@ }, "Microsoft.Data.SqlClient.SNI.runtime": { "type": "Transitive", - "resolved": "3.0.0", - "contentHash": "n1sNyjJgu2pYWKgw3ZPikw3NiRvG4kt7Ya5MK8u77Rgj/1bTFqO/eDF4k5W9H5GXplMZCpKkNbp5kNBICgSB0w==" + "resolved": "5.0.1", + "contentHash": "y0X5MxiNdbITJYoafJ2ruaX6hqO0twpCGR/ipiDOe85JKLU8WL4TuAQfDe5qtt3bND5Je26HnrarLSAMMnVTNg==" }, "Microsoft.DotNet.PlatformAbstractions": { "type": "Transitive", @@ -465,10 +464,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", + "resolved": "2.2.0", + "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -508,16 +507,16 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "MgYpU5cwZohUMKKg3sbPhvGG+eAZ/59E9UwPwlrUkyXU+PGzqwZg9yyQNjhxuAWmoNoFReoemeCku50prYSGzA==" + "resolved": "2.2.0", + "contentHash": "f9hstgjVmr6rmrfGSpfsVOl2irKAgr1QjrSi3FgnS7kulxband50f2brRLwySAQTADPZeTdow0mpSMcoAdadCw==" }, "Microsoft.Extensions.DependencyModel": { "type": "Transitive", @@ -533,10 +532,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", + "resolved": "2.2.0", + "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -567,13 +566,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", + "resolved": "2.2.0", + "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0" } }, "Microsoft.Extensions.Logging": { @@ -589,8 +588,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" + "resolved": "2.2.0", + "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -603,16 +602,17 @@ }, "Microsoft.Extensions.ObjectPool": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "SErON45qh4ogDp6lr6UvVmFYW0FERihW+IQ+2JyFv1PUyWktcJytFaWH5zarufJvZwhci7Rf1IyGXr9pVEadTw==" + "resolved": "2.2.0", + "contentHash": "gA8H7uQOnM5gb+L0uTNjViHYr+hRDqCdfugheGo/MxQnuHzmhhzCBTIPm19qL1z1Xe0NEMabfcOBGv9QghlZ8g==" }, "Microsoft.Extensions.Options": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "V7lXCU78lAbzaulCGFKojcCyG8RTJicEbiBkPJjFqiqXwndEBBIehdXRMWEVU3UtzQ1yDvphiWUL9th6/4gJ7w==", + "resolved": "2.2.0", + "contentHash": "UpZLNLBpIZ0GTebShui7xXYh6DmBHjWM8NxGxZbdQh/bPZ5e6YswqI+bru6BnEL5eWiOdodsXtEz3FROcgi/qg==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.1", - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.Primitives": "2.2.0", + "System.ComponentModel.Annotations": "4.5.0" } }, "Microsoft.Extensions.Options.ConfigurationExtensions": { @@ -628,8 +628,8 @@ }, "Microsoft.Extensions.Primitives": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "scJ1GZNIxMmjpENh0UZ8XCQ6vzr/LzeF9WvEA51Ix2OQGAs9WPgPu8ABVUdvpKPLuor/t05gm6menJK3PwqOXg==", + "resolved": "2.2.0", + "contentHash": "azyQtqbm4fSaDzZHD/J+V6oWMFaf2tWP4WEGIYePLCMw3+b2RQdj9ybgbQyjCshcitQKQ4lEDOZjmSlTTrHxUg==", "dependencies": { "System.Memory": "4.5.1", "System.Runtime.CompilerServices.Unsafe": "4.5.1" @@ -637,78 +637,94 @@ }, "Microsoft.Identity.Client": { "type": "Transitive", - "resolved": "4.30.1", - "contentHash": "xk8tJeGfB2yD3+d7a0DXyV7/HYyEG10IofUHYHoPYKmDbroi/j9t1BqSHgbq1nARDjg7m8Ki6e21AyNU7e/R4Q==" + "resolved": "4.45.0", + "contentHash": "ircobISCLWbtE5eEoLKU+ldfZ8O41vg4lcy38KRj/znH17jvBiAl8oxcyNp89CsuqE3onxIpn21Ca7riyDDrRw==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.18.0" + } }, "Microsoft.Identity.Client.Extensions.Msal": { "type": "Transitive", - "resolved": "2.18.4", - "contentHash": "HpG4oLwhQsy0ce7OWq9iDdLtJKOvKRStIKoSEOeBMKuohfuOWNDyhg8fMAJkpG/kFeoe4J329fiMHcJmmB+FPw==", + "resolved": "2.19.3", + "contentHash": "zVVZjn8aW7W79rC1crioDgdOwaFTQorsSO6RgVlDDjc7MvbEGz071wSNrjVhzR0CdQn6Sefx7Abf1o7vasmrLg==", "dependencies": { - "Microsoft.Identity.Client": "4.30.0", + "Microsoft.Identity.Client": "4.38.0", "System.Security.Cryptography.ProtectedData": "4.5.0" } }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "6.21.0", + "contentHash": "XeE6LQtD719Qs2IG7HDi1TSw9LIkDbJ33xFiOBoHbApVw/8GpIBCbW+t7RwOjErUDyXZvjhZliwRkkLb8Z1uzg==" + }, "Microsoft.IdentityModel.JsonWebTokens": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "+7JIww64PkMt7NWFxoe4Y/joeF7TAtA/fQ0b2GFGcagzB59sKkTt/sMZWR6aSZht5YC7SdHi3W6yM1yylRGJCQ==", + "resolved": "6.21.0", + "contentHash": "d3h1/BaMeylKTkdP6XwRCxuOoDJZ44V9xaXr6gl5QxmpnZGdoK3bySo3OQN8ehRLJHShb94ElLUvoXyglQtgAw==", "dependencies": { - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "Microsoft.IdentityModel.Logging": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "Rfh/p4MaN4gkmhPxwbu8IjrmoDncGfHHPh1sTnc0AcM/Oc39/fzC9doKNWvUAjzFb8LqA6lgZyblTrIsX/wDXg==" + "resolved": "6.21.0", + "contentHash": "tuEhHIQwvBEhMf8I50hy8FHmRSUkffDFP5EdLsSDV4qRcl2wvOPkQxYqEzWkh+ytW6sbdJGEXElGhmhDfAxAKg==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "6.21.0" + } }, "Microsoft.IdentityModel.Protocols": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "OJZx5nPdiH+MEkwCkbJrTAUiO/YzLe0VSswNlDxJsJD9bhOIdXHufh650pfm59YH1DNevp3/bXzukKrG57gA1w==", + "resolved": "6.21.0", + "contentHash": "0FqY5cTLQKtHrClzHEI+QxJl8OBT2vUiEQQB7UKk832JDiJJmetzYZ3AdSrPjN/3l3nkhByeWzXnhrX0JbifKg==", "dependencies": { - "Microsoft.IdentityModel.Logging": "6.8.0", - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.Logging": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "Microsoft.IdentityModel.Protocols.OpenIdConnect": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "X/PiV5l3nYYsodtrNMrNQIVlDmHpjQQ5w48E+o/D5H4es2+4niEyQf3l03chvZGWNzBRhfSstaXr25/Ye4AeYw==", + "resolved": "6.21.0", + "contentHash": "vtSKL7n6EnAsLyxmiviusm6LKrblT2ndnNqN6rvVq6iIHAnPCK9E2DkDx6h1Jrpy1cvbp40r0cnTg23nhEAGTA==", "dependencies": { - "Microsoft.IdentityModel.Protocols": "6.8.0", - "System.IdentityModel.Tokens.Jwt": "6.8.0" + "Microsoft.IdentityModel.Protocols": "6.21.0", + "System.IdentityModel.Tokens.Jwt": "6.21.0" } }, "Microsoft.IdentityModel.Tokens": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "gTqzsGcmD13HgtNePPcuVHZ/NXWmyV+InJgalW/FhWpII1D7V1k0obIseGlWMeA4G+tZfeGMfXr0klnWbMR/mQ==", + "resolved": "6.21.0", + "contentHash": "AAEHZvZyb597a+QJSmtxH3n2P1nIJGpZ4Q89GTenknRx6T6zyfzf592yW/jA5e8EHN4tNMjjXHQaYWEq5+L05w==", "dependencies": { "Microsoft.CSharp": "4.5.0", - "Microsoft.IdentityModel.Logging": "6.8.0", + "Microsoft.IdentityModel.Logging": "6.21.0", "System.Security.Cryptography.Cng": "4.5.0" } }, "Microsoft.Net.Http.Headers": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "lPNIphl8b2EuhOE9dMH6EZDmu7pS882O+HMi5BJNsigxHaWlBrYxZHFZgE18cyaPp6SSZcTkKkuzfjV/RRQKlA==", + "resolved": "2.2.0", + "contentHash": "iZNkjYqlo8sIOI0bQfpsSoMTmB/kyvmV2h225ihyZT33aTp48ZpF6qYnXxzSXmHt8DpBAwBTX+1s1UFLbYfZKg==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1", + "Microsoft.Extensions.Primitives": "2.2.0", "System.Buffers": "4.5.0" } }, "Microsoft.NETCore.Platforms": { "type": "Transitive", - "resolved": "3.1.0", - "contentHash": "z7aeg8oHln2CuNulfhiLYxCVMPEwBl3rzicjvIX+4sUuCwvXw5oXQEtbiU2c0z4qYL5L3Kmx0mMA/+t/SbY67w==" + "resolved": "5.0.0", + "contentHash": "VyPlqzH2wavqquTcYpkIIAQ6WdenuKoFN0BdYBbCWsclXacSOHNQn66Gt4z5NBqEYW0FAPm5rlvki9ZiCij5xQ==" }, "Microsoft.NETCore.Targets": { "type": "Transitive", "resolved": "1.1.0", "contentHash": "aOZA3BWfz9RXjpzt0sRJJMjAscAUm3Hoa4UWAfceV9UTYxgwZ1lZt5nO2myFf+/jetYQo4uTP7zS8sJY67BBxg==" }, + "Microsoft.SqlServer.Server": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "N4KeF3cpcm1PUHym1RmakkzfkEv3GRMyofVv40uXsQhCQeglr2OHNcUk2WOG51AKpGO8ynGpo9M/kFXSzghwug==" + }, "Microsoft.TestPlatform.ObjectModel": { "type": "Transitive", "resolved": "17.0.0", @@ -739,19 +755,19 @@ }, "Microsoft.Win32.Registry": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "KSrRMb5vNi0CWSGG1++id2ZOs/1QhRqROt+qgbEAdQuGjGrFcl4AOl4/exGPUYz2wUnU42nvJqon1T3U0kPXLA==", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", "dependencies": { - "System.Security.AccessControl": "4.7.0", - "System.Security.Principal.Windows": "4.7.0" + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" } }, "Microsoft.Win32.SystemEvents": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "mtVirZr++rq+XCDITMUdnETD59XoeMxSpLRIII7JRI6Yj0LEDiO1pPn0ktlnIj12Ix8bfvQqQDMMIF9wC98oCA==", + "resolved": "5.0.0", + "contentHash": "Bh6blKG8VAKvXiLe2L+sEsn62nc1Ij34MrNxepD2OCrS5cpCwQa9MeLyhVQPQ/R4Wlzwuy6wMK8hLb11QPDRsQ==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0" + "Microsoft.NETCore.Platforms": "5.0.0" } }, "ncrontab.signed": { @@ -1016,8 +1032,8 @@ }, "System.ComponentModel.Annotations": { "type": "Transitive", - "resolved": "4.4.0", - "contentHash": "29K3DQ+IGU7LBaMjTo7SI7T7X/tsMtLvz1p56LJ556Iu0Dw3pKZw5g8yCYCWMRxrOF0Hr0FU0FwW0o42y2sb3A==" + "resolved": "4.5.0", + "contentHash": "UxYQ3FGUOtzJ7LfSdnYSFd7+oEv6M8NgUatatIN2HxNtDdlcvFAf+VIq4Of9cDMJEJC0aSRv/x898RYhB4Yppg==" }, "System.ComponentModel.Primitives": { "type": "Transitive", @@ -1053,11 +1069,11 @@ }, "System.Configuration.ConfigurationManager": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "/anOTeSZCNNI2zDilogWrZ8pNqCmYbzGNexUnNhjW8k0sHqEZ2nHJBp147jBV3hGYswu5lINpNg1vxR7bnqvVA==", + "resolved": "5.0.0", + "contentHash": "aM7cbfEfVNlEEOj3DsZP+2g9NRwbkyiAv2isQEzw7pnkDg9ekCU2m1cdJLM02Uq691OaCS91tooaxcEn8d0q5w==", "dependencies": { - "System.Security.Cryptography.ProtectedData": "4.7.0", - "System.Security.Permissions": "4.7.0" + "System.Security.Cryptography.ProtectedData": "5.0.0", + "System.Security.Permissions": "5.0.0" } }, "System.Console": { @@ -1084,8 +1100,8 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "oJjw3uFuVDJiJNbCD8HB4a2p3NYLdt1fiT5OGsPLw+WTOuG0KpP4OXelMmmVKpClueMsit6xOlzy4wNKQFiBLg==" + "resolved": "5.0.0", + "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==" }, "System.Diagnostics.Tools": { "type": "Transitive", @@ -1123,15 +1139,6 @@ "System.Runtime": "4.3.0" } }, - "System.Drawing.Common": { - "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "v+XbyYHaZjDfn0ENmJEV1VYLgGgCTx1gnfOBcppowbpOAriglYgGCvFCPr2EEZyBvXlpxbEsTwkOlInl107ahA==", - "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0", - "Microsoft.Win32.SystemEvents": "4.7.0" - } - }, "System.Dynamic.Runtime": { "type": "Transitive", "resolved": "4.3.0", @@ -1153,6 +1160,11 @@ "System.Threading": "4.3.0" } }, + "System.Formats.Asn1": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "MTvUIktmemNB+El0Fgw9egyqT9AYSIk6DTJeoDSpc3GIHxHCMo8COqkWT1mptX5tZ1SlQ6HJZ0OsSvMth1c12w==" + }, "System.Globalization": { "type": "Transitive", "resolved": "4.3.0", @@ -1189,11 +1201,11 @@ }, "System.IdentityModel.Tokens.Jwt": { "type": "Transitive", - "resolved": "6.8.0", - "contentHash": "5tBCjAub2Bhd5qmcd0WhR5s354e4oLYa//kOWrkX+6/7ZbDDJjMTfwLSOiZ/MMpWdE4DWPLOfTLOq/juj9CKzA==", + "resolved": "6.21.0", + "contentHash": "JRD8AuypBE+2zYxT3dMJomQVsPYsCqlyZhWel3J1d5nzQokSRyTueF+Q4ID3Jcu6zSZKuzOdJ1MLTkbQsDqcvQ==", "dependencies": { - "Microsoft.IdentityModel.JsonWebTokens": "6.8.0", - "Microsoft.IdentityModel.Tokens": "6.8.0" + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Tokens": "6.21.0" } }, "System.IO": { @@ -1569,11 +1581,11 @@ }, "System.Security.AccessControl": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "JECvTt5aFF3WT3gHpfofL2MNNP6v84sxtXxpqhLBCcDRzqsPBmHhQ6shv4DwwN2tRlzsUxtb3G9M3763rbXKDg==", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0", - "System.Security.Principal.Windows": "4.7.0" + "Microsoft.NETCore.Platforms": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" } }, "System.Security.Cryptography.Algorithms": { @@ -1599,8 +1611,11 @@ }, "System.Security.Cryptography.Cng": { "type": "Transitive", - "resolved": "4.5.0", - "contentHash": "WG3r7EyjUe9CMPFSs6bty5doUqT+q9pbI80hlNzo2SkPkZ4VTuZkGWjpp77JB8+uaL4DFPRdBsAY+DX3dBK92A==" + "resolved": "5.0.0", + "contentHash": "jIMXsKn94T9JY7PvPq/tMfqa6GAaHpElRDpmG+SuL+D3+sTw2M8VhnibKnN8Tq+4JqbPJ/f+BwtLeDMEnzAvRg==", + "dependencies": { + "System.Formats.Asn1": "5.0.0" + } }, "System.Security.Cryptography.Csp": { "type": "Transitive", @@ -1677,8 +1692,8 @@ }, "System.Security.Cryptography.ProtectedData": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "ehYW0m9ptxpGWvE4zgqongBVWpSDU/JCFD4K7krxkQwSz/sFQjEXCUqpvencjy6DYDbn7Ig09R8GFffu8TtneQ==" + "resolved": "5.0.0", + "contentHash": "HGxMSAFAPLNoxBvSfW08vHde0F9uh7BjASwu6JF9JnXuEPhCY3YUqURn0+bQV/4UWeaqymmrHWV+Aw9riQCtCA==" }, "System.Security.Cryptography.X509Certificates": { "type": "Transitive", @@ -1714,17 +1729,17 @@ }, "System.Security.Permissions": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "dkOV6YYVBnYRa15/yv004eCGRBVADXw8qRbbNiCn/XpdJSUXkkUeIvdvFHkvnko4CdKMqG8yRHC4ox83LSlMsQ==", + "resolved": "5.0.0", + "contentHash": "uE8juAhEkp7KDBCdjDIE3H9R1HJuEHqeqX8nLX9gmYKWwsqk3T5qZlPx8qle5DPKimC/Fy3AFTdV7HamgCh9qQ==", "dependencies": { - "System.Security.AccessControl": "4.7.0", - "System.Windows.Extensions": "4.7.0" + "System.Security.AccessControl": "5.0.0", + "System.Windows.Extensions": "5.0.0" } }, "System.Security.Principal.Windows": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "ojD0PX0XhneCsUbAZVKdb7h/70vyYMDYs85lwEI+LngEONe/17A0cFaRFqZU+sOEidcVswYWikYOQ9PPfjlbtQ==" + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" }, "System.Text.Encoding": { "type": "Transitive", @@ -1738,10 +1753,10 @@ }, "System.Text.Encoding.CodePages": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "aeu4FlaUTemuT1qOd1MyU4T516QR4Fy+9yDbwWMPHOHy7U8FD6SgTzdZFO7gHcfAPHtECqInbwklVvUK4RHcNg==", + "resolved": "5.0.0", + "contentHash": "NyscU59xX6Uo91qvhOs2Ccho3AR2TnZPomo1Z0K6YpyztBPM/A5VbkzOO19sy3A3i1TtEnTxA7bCe3Us+r5MWg==", "dependencies": { - "Microsoft.NETCore.Platforms": "3.1.0" + "Microsoft.NETCore.Platforms": "5.0.0" } }, "System.Text.Encoding.Extensions": { @@ -1762,8 +1777,8 @@ }, "System.Text.Json": { "type": "Transitive", - "resolved": "4.6.0", - "contentHash": "4F8Xe+JIkVoDJ8hDAZ7HqLkjctN/6WItJIzQaifBwClC7wmoLSda/Sv2i6i1kycqDb3hWF4JCVbpAweyOKHEUA==" + "resolved": "4.7.2", + "contentHash": "TcMd95wcrubm9nHvJEQs70rC0H/8omiSGGpU4FQ/ZA1URIqD4pjmFJh2Mfv1yH1eHgJDWTi2hMDXwTET+zOOyg==" }, "System.Text.RegularExpressions": { "type": "Transitive", @@ -1799,8 +1814,8 @@ }, "System.Threading.Tasks.Extensions": { "type": "Transitive", - "resolved": "4.5.2", - "contentHash": "BG/TNxDFv0svAzx8OiMXDlsHfGw623BZ8tCXw4YLhDFDvDhNUEV58jKYMGRnkbJNm7c3JNNJDiN7JBMzxRBR2w==" + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==" }, "System.Threading.Timer": { "type": "Transitive", @@ -1814,10 +1829,10 @@ }, "System.Windows.Extensions": { "type": "Transitive", - "resolved": "4.7.0", - "contentHash": "CeWTdRNfRaSh0pm2gDTJFwVaXfTq6Xwv/sA887iwPTneW7oMtMlpvDIO+U60+3GWTB7Aom6oQwv5VZVUhQRdPQ==", + "resolved": "5.0.0", + "contentHash": "c1ho9WU9ZxMZawML+ssPKZfdnrg/OjR3pe0m9v8230z3acqphwvPJqzAkH54xRYm5ntZHGG1EPP3sux9H3qSPg==", "dependencies": { - "System.Drawing.Common": "4.7.0" + "System.Drawing.Common": "5.0.0" } }, "System.Xml.ReaderWriter": { @@ -1930,42 +1945,41 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.14.0, )", - "Microsoft.AspNetCore.Http": "[2.1.22, )", - "Microsoft.Azure.WebJobs": "[3.0.31, )", - "Microsoft.Data.SqlClient": "[3.0.1, )", - "Newtonsoft.Json": "[11.0.2, )", - "System.Runtime.Caching": "[4.7.0, )", + "Microsoft.ApplicationInsights": "[2.17.0, )", + "Microsoft.AspNetCore.Http": "[2.2.2, )", + "Microsoft.Azure.WebJobs": "[3.0.32, )", + "Microsoft.Data.SqlClient": "[5.0.1, )", + "Newtonsoft.Json": "[13.0.1, )", + "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } }, "microsoft.azure.webjobs.extensions.sql.samples": { "type": "Project", "dependencies": { - "Microsoft.AspNetCore.Http": "[2.1.22, )", + "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )", - "Microsoft.NET.Sdk.Functions": "[3.1.1, )", - "Newtonsoft.Json": "[11.0.2, )" + "Microsoft.NET.Sdk.Functions": "[4.1.3, )", + "Newtonsoft.Json": "[13.0.1, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.14.0, )", - "resolved": "2.14.0", - "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", + "requested": "[2.17.0, )", + "resolved": "2.17.0", + "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4" + "System.Diagnostics.DiagnosticSource": "5.0.0" } }, "Microsoft.Azure.WebJobs": { "type": "CentralTransitive", - "requested": "[3.0.31, )", - "resolved": "3.0.31", - "contentHash": "Jn6E7OgT7LkwVB6lCpjXJcoQIvKrbJT+taVLA4CekEpa21pzZv6nQ2sYRSNzPz5ul3FAcYhmrCQgV7v2iopjgA==", + "requested": "[3.0.32, )", + "resolved": "3.0.32", + "contentHash": "uN8GsFqPFHHcSrwwj/+0tGe6F6cOwugqUiePPw7W3TL9YC594+Hw8GBK5S/fcDWXacqvRRGf9nDX8xP94/Yiyw==", "dependencies": { - "Microsoft.Azure.WebJobs.Core": "3.0.31", + "Microsoft.Azure.WebJobs.Core": "3.0.32", "Microsoft.Extensions.Configuration": "2.1.1", "Microsoft.Extensions.Configuration.Abstractions": "2.1.1", "Microsoft.Extensions.Configuration.EnvironmentVariables": "2.1.0", @@ -1991,30 +2005,35 @@ }, "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": { "type": "CentralTransitive", - "requested": "[1.2.3, )", - "resolved": "1.2.2", - "contentHash": "vpiNt3JM1pt/WrDIkg7G2DHhIpI4t5I+R9rmXCxIGiby5oPGEolyfiYZdEf2kMMN3SbWzVAbk4Q3jKgFhO9MaQ==", + "requested": "[4.0.1, )", + "resolved": "4.0.1", + "contentHash": "o1E0hetLv8Ix0teA1hGH9D136RGSs24Njm5+a4FKzJHLlxfclvmOxmcg87vcr6LIszKzenNKd1oJGnOwg2WMnw==", "dependencies": { "System.Runtime.Loader": "4.3.0" } }, "Microsoft.Data.SqlClient": { "type": "CentralTransitive", - "requested": "[3.0.1, )", - "resolved": "3.0.1", - "contentHash": "5Jgcds8yukUeOHvc8S0rGW87rs2uYEM9/YyrYIb/0C+vqzIa2GiqbVPCDVcnApWhs67OSXLTM7lO6jro24v/rA==", - "dependencies": { - "Azure.Identity": "1.3.0", - "Microsoft.Data.SqlClient.SNI.runtime": "3.0.0", - "Microsoft.Identity.Client": "4.22.0", - "Microsoft.IdentityModel.JsonWebTokens": "6.8.0", - "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.8.0", - "Microsoft.Win32.Registry": "4.7.0", - "System.Configuration.ConfigurationManager": "4.7.0", - "System.Diagnostics.DiagnosticSource": "4.7.0", - "System.Runtime.Caching": "4.7.0", - "System.Security.Principal.Windows": "4.7.0", - "System.Text.Encoding.CodePages": "4.7.0", + "requested": "[5.0.1, )", + "resolved": "5.0.1", + "contentHash": "uu8dfrsx081cSbEevWuZAvqdmANDGJkbLBL2G3j0LAZxX1Oy8RCVAaC4Lcuak6jNicWP6CWvHqBTIEmQNSxQlw==", + "dependencies": { + "Azure.Identity": "1.6.0", + "Microsoft.Data.SqlClient.SNI.runtime": "5.0.1", + "Microsoft.Identity.Client": "4.45.0", + "Microsoft.IdentityModel.JsonWebTokens": "6.21.0", + "Microsoft.IdentityModel.Protocols.OpenIdConnect": "6.21.0", + "Microsoft.SqlServer.Server": "1.0.0", + "Microsoft.Win32.Registry": "5.0.0", + "System.Buffers": "4.5.1", + "System.Configuration.ConfigurationManager": "5.0.0", + "System.Diagnostics.DiagnosticSource": "5.0.0", + "System.IO": "4.3.0", + "System.Resources.ResourceManager": "4.3.0", + "System.Runtime.Caching": "5.0.0", + "System.Security.Cryptography.Cng": "5.0.0", + "System.Security.Principal.Windows": "5.0.0", + "System.Text.Encoding.CodePages": "5.0.0", "System.Text.Encodings.Web": "4.7.2" } }, @@ -2024,13 +2043,22 @@ "resolved": "3.3.2", "contentHash": "MQc8GppZJLmjvcpEdf3EkC6ovsp7gRWt2e5mC7dcIOrgwSc+yjFd3JQ0iRqr3XrUT6rb/phv0IkEmBtbfVA7AQ==" }, + "System.Drawing.Common": { + "type": "CentralTransitive", + "requested": "[5.0.3, )", + "resolved": "5.0.0", + "contentHash": "SztFwAnpfKC8+sEKXAFxCBWhKQaEd97EiOL7oZJZP56zbqnLpmxACWA8aGseaUExciuEAUuR9dY8f7HkTRAdnw==", + "dependencies": { + "Microsoft.Win32.SystemEvents": "5.0.0" + } + }, "System.Runtime.Caching": { "type": "CentralTransitive", - "requested": "[4.7.0, )", - "resolved": "4.7.0", - "contentHash": "NdvNRjTPxYvIEhXQszT9L9vJhdQoX6AQ0AlhjTU+5NqFQVuacJTfhPVAvtGWNA2OJCqRiR/okBcZgMwI6MqcZg==", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "30D6MkO8WF9jVGWZIP0hmCN8l9BTY4LCsAzLIe4xFSXzs+AjDotR7DpSmj27pFskDURzUvqYYY0ikModgBTxWw==", "dependencies": { - "System.Configuration.ConfigurationManager": "4.7.0" + "System.Configuration.ConfigurationManager": "5.0.0" } } } From e46dc4293599f1854a48954adac8ec027824d0c5 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 28 Oct 2022 10:31:37 -0700 Subject: [PATCH 55/77] Use lower Functions SDK package --- Directory.Packages.props | 6 +- performance/SqlBindingBenchmarks.cs | 4 +- performance/SqlTriggerBindingPerformance.cs | 2 +- performance/packages.lock.json | 246 ++++++++++---------- samples/samples-csharp/packages.lock.json | 238 +++++++++---------- src/packages.lock.json | 22 +- test/packages.lock.json | 242 +++++++++---------- 7 files changed, 380 insertions(+), 380 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 3945520c6..c5b581551 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,9 +1,9 @@ - + - + @@ -15,7 +15,7 @@ - + diff --git a/performance/SqlBindingBenchmarks.cs b/performance/SqlBindingBenchmarks.cs index 7917e17e2..77e4e7b50 100644 --- a/performance/SqlBindingBenchmarks.cs +++ b/performance/SqlBindingBenchmarks.cs @@ -9,8 +9,8 @@ public class SqlBindingPerformance { public static void Main() { - BenchmarkRunner.Run(); - BenchmarkRunner.Run(); + //BenchmarkRunner.Run(); + //BenchmarkRunner.Run(); BenchmarkRunner.Run(); } } diff --git a/performance/SqlTriggerBindingPerformance.cs b/performance/SqlTriggerBindingPerformance.cs index 43a63f5ea..d96c3a115 100644 --- a/performance/SqlTriggerBindingPerformance.cs +++ b/performance/SqlTriggerBindingPerformance.cs @@ -22,7 +22,7 @@ public void GlobalSetup() [Arguments(1)] [Arguments(10)] [Arguments(100)] - [Arguments(1000)] + // [Arguments(1000)] public async Task ProductsTriggerTest(int count) { await this.WaitForProductChanges( diff --git a/performance/packages.lock.json b/performance/packages.lock.json index cf3686792..9266bc760 100644 --- a/performance/packages.lock.json +++ b/performance/packages.lock.json @@ -114,8 +114,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.8", - "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", + "resolved": "5.2.4", + "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -123,59 +123,59 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", + "resolved": "2.1.0", + "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", + "resolved": "2.1.0", + "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http": "2.2.0", - "Microsoft.AspNetCore.Http.Extensions": "2.2.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Http.Extensions": "2.1.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", + "resolved": "2.1.0", + "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", + "resolved": "2.1.0", + "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Authorization": "2.2.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Authorization": "2.1.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", + "resolved": "2.1.0", + "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", + "resolved": "2.1.0", + "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.2.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Http.Features": "2.1.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { @@ -189,12 +189,12 @@ }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", + "resolved": "2.1.0", + "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Net.Http.Headers": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Net.Http.Headers": "2.1.0", "System.Buffers": "4.5.0" } }, @@ -208,8 +208,8 @@ }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", + "resolved": "2.1.0", + "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -217,81 +217,80 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", + "resolved": "2.1.0", + "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Net.Http.Headers": "2.2.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", + "Microsoft.Net.Http.Headers": "2.1.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.2.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http": "2.2.0", - "Microsoft.AspNetCore.Http.Extensions": "2.2.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Routing": "2.2.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Extensions.DependencyInjection": "2.2.0", + "resolved": "2.1.0", + "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.1.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Http.Extensions": "2.1.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Routing": "2.1.0", + "Microsoft.Extensions.DependencyInjection": "2.1.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.1" + "System.Threading.Tasks.Extensions": "4.5.0" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", + "resolved": "2.1.0", + "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.2.0", - "Microsoft.AspNetCore.Mvc.Core": "2.2.0" + "Microsoft.AspNetCore.JsonPatch": "2.1.0", + "Microsoft.AspNetCore.Mvc.Core": "2.1.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", + "resolved": "2.1.0", + "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.6", - "Microsoft.AspNetCore.Mvc.Core": "2.2.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", - "Microsoft.AspNetCore.WebUtilities": "2.2.0" + "Microsoft.AspNet.WebApi.Client": "5.2.4", + "Microsoft.AspNetCore.Mvc.Core": "2.1.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", + "Microsoft.AspNetCore.WebUtilities": "2.1.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", + "resolved": "2.1.0", + "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.2.2", - "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", + "resolved": "2.1.0", + "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.2.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.ObjectPool": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.AspNetCore.Http.Extensions": "2.1.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.ObjectPool": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", + "resolved": "2.1.0", + "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.WebUtilities": { @@ -329,15 +328,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.2.0", - "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", + "resolved": "3.0.2", + "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.8", - "Microsoft.AspNetCore.Http": "2.2.2", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", - "Microsoft.AspNetCore.Routing": "2.2.2", - "Microsoft.Azure.WebJobs": "3.0.32" + "Microsoft.AspNet.WebApi.Client": "5.2.4", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", + "Microsoft.AspNetCore.Routing": "2.1.0", + "Microsoft.Azure.WebJobs": "3.0.2" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -507,10 +506,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", + "resolved": "2.1.1", + "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.1" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -550,10 +549,10 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", + "resolved": "2.1.0", + "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { @@ -575,10 +574,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", + "resolved": "2.1.0", + "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -609,13 +608,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", + "resolved": "2.1.0", + "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0" } }, "Microsoft.Extensions.Logging": { @@ -631,8 +630,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" + "resolved": "2.1.1", + "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -2103,11 +2102,11 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.17.0, )", + "Microsoft.ApplicationInsights": "[2.14.0, )", "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs": "[3.0.32, )", "Microsoft.Data.SqlClient": "[5.0.1, )", - "Newtonsoft.Json": "[13.0.1, )", + "Newtonsoft.Json": "[11.0.2, )", "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } @@ -2118,8 +2117,8 @@ "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )", - "Microsoft.NET.Sdk.Functions": "[4.1.3, )", - "Newtonsoft.Json": "[13.0.1, )" + "Microsoft.NET.Sdk.Functions": "[4.1.1, )", + "Newtonsoft.Json": "[11.0.2, )" } }, "microsoft.azure.webjobs.extensions.sql.tests": { @@ -2128,21 +2127,22 @@ "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Sql.Samples": "[1.0.0, )", - "Microsoft.NET.Sdk.Functions": "[4.1.3, )", + "Microsoft.NET.Sdk.Functions": "[4.1.1, )", "Microsoft.NET.Test.Sdk": "[17.0.0, )", "Moq": "[4.14.3, )", - "Newtonsoft.Json": "[13.0.1, )", + "Newtonsoft.Json": "[11.0.2, )", "xunit": "[2.4.0, )", "xunit.runner.visualstudio": "[2.4.0, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.17.0, )", - "resolved": "2.17.0", - "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", + "requested": "[2.14.0, )", + "resolved": "2.14.0", + "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "5.0.0" + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory": "4.5.4" } }, "Microsoft.AspNetCore.Http": { @@ -2224,16 +2224,16 @@ }, "Microsoft.NET.Sdk.Functions": { "type": "CentralTransitive", - "requested": "[4.1.3, )", - "resolved": "4.1.3", - "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", + "requested": "[4.1.1, )", + "resolved": "4.1.1", + "contentHash": "G+b+bHtta7P8KPItSWygbVwQhwamU/WWmBoiv3snf8ScVYai3PbC1JSW3H22X+askqxhiw/Tx0yZKdE5oKMhRQ==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", - "Newtonsoft.Json": "13.0.1" + "Newtonsoft.Json": "11.0.2" } }, "Microsoft.NET.Test.Sdk": { @@ -2264,9 +2264,9 @@ }, "Newtonsoft.Json": { "type": "CentralTransitive", - "requested": "[13.0.1, )", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + "requested": "[11.0.2, )", + "resolved": "11.0.2", + "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" }, "System.Drawing.Common": { "type": "CentralTransitive", diff --git a/samples/samples-csharp/packages.lock.json b/samples/samples-csharp/packages.lock.json index 6cc85cff0..fbf18e49f 100644 --- a/samples/samples-csharp/packages.lock.json +++ b/samples/samples-csharp/packages.lock.json @@ -27,23 +27,23 @@ }, "Microsoft.NET.Sdk.Functions": { "type": "Direct", - "requested": "[4.1.3, )", - "resolved": "4.1.3", - "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", + "requested": "[4.1.1, )", + "resolved": "4.1.1", + "contentHash": "G+b+bHtta7P8KPItSWygbVwQhwamU/WWmBoiv3snf8ScVYai3PbC1JSW3H22X+askqxhiw/Tx0yZKdE5oKMhRQ==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", - "Newtonsoft.Json": "13.0.1" + "Newtonsoft.Json": "11.0.2" } }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[13.0.1, )", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + "requested": "[11.0.2, )", + "resolved": "11.0.2", + "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" }, "Azure.Core": { "type": "Transitive", @@ -102,8 +102,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.8", - "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", + "resolved": "5.2.4", + "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -111,59 +111,59 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", + "resolved": "2.1.0", + "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", + "resolved": "2.1.0", + "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http": "2.2.0", - "Microsoft.AspNetCore.Http.Extensions": "2.2.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Http.Extensions": "2.1.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", + "resolved": "2.1.0", + "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", + "resolved": "2.1.0", + "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Authorization": "2.2.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Authorization": "2.1.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", + "resolved": "2.1.0", + "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", + "resolved": "2.1.0", + "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.2.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Http.Features": "2.1.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { @@ -177,12 +177,12 @@ }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", + "resolved": "2.1.0", + "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Net.Http.Headers": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Net.Http.Headers": "2.1.0", "System.Buffers": "4.5.0" } }, @@ -196,8 +196,8 @@ }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", + "resolved": "2.1.0", + "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -205,81 +205,80 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", + "resolved": "2.1.0", + "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Net.Http.Headers": "2.2.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", + "Microsoft.Net.Http.Headers": "2.1.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.2.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http": "2.2.0", - "Microsoft.AspNetCore.Http.Extensions": "2.2.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Routing": "2.2.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Extensions.DependencyInjection": "2.2.0", + "resolved": "2.1.0", + "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.1.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Http.Extensions": "2.1.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Routing": "2.1.0", + "Microsoft.Extensions.DependencyInjection": "2.1.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.1" + "System.Threading.Tasks.Extensions": "4.5.0" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", + "resolved": "2.1.0", + "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.2.0", - "Microsoft.AspNetCore.Mvc.Core": "2.2.0" + "Microsoft.AspNetCore.JsonPatch": "2.1.0", + "Microsoft.AspNetCore.Mvc.Core": "2.1.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", + "resolved": "2.1.0", + "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.6", - "Microsoft.AspNetCore.Mvc.Core": "2.2.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", - "Microsoft.AspNetCore.WebUtilities": "2.2.0" + "Microsoft.AspNet.WebApi.Client": "5.2.4", + "Microsoft.AspNetCore.Mvc.Core": "2.1.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", + "Microsoft.AspNetCore.WebUtilities": "2.1.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", + "resolved": "2.1.0", + "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.2.2", - "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", + "resolved": "2.1.0", + "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.2.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.ObjectPool": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.AspNetCore.Http.Extensions": "2.1.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.ObjectPool": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", + "resolved": "2.1.0", + "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.WebUtilities": { @@ -317,15 +316,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.2.0", - "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", + "resolved": "3.0.2", + "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.8", - "Microsoft.AspNetCore.Http": "2.2.2", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", - "Microsoft.AspNetCore.Routing": "2.2.2", - "Microsoft.Azure.WebJobs": "3.0.32" + "Microsoft.AspNet.WebApi.Client": "5.2.4", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", + "Microsoft.AspNetCore.Routing": "2.1.0", + "Microsoft.Azure.WebJobs": "3.0.2" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -412,10 +411,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", + "resolved": "2.1.1", + "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.1" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -455,10 +454,10 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", + "resolved": "2.1.0", + "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { @@ -480,10 +479,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", + "resolved": "2.1.0", + "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -514,13 +513,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", + "resolved": "2.1.0", + "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0" } }, "Microsoft.Extensions.Logging": { @@ -536,8 +535,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" + "resolved": "2.1.1", + "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -1742,22 +1741,23 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.17.0, )", + "Microsoft.ApplicationInsights": "[2.14.0, )", "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs": "[3.0.32, )", "Microsoft.Data.SqlClient": "[5.0.1, )", - "Newtonsoft.Json": "[13.0.1, )", + "Newtonsoft.Json": "[11.0.2, )", "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.17.0, )", - "resolved": "2.17.0", - "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", + "requested": "[2.14.0, )", + "resolved": "2.14.0", + "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "5.0.0" + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory": "4.5.4" } }, "Microsoft.Azure.WebJobs": { diff --git a/src/packages.lock.json b/src/packages.lock.json index 220fd768e..ea176fd93 100644 --- a/src/packages.lock.json +++ b/src/packages.lock.json @@ -4,11 +4,12 @@ ".NETStandard,Version=v2.0": { "Microsoft.ApplicationInsights": { "type": "Direct", - "requested": "[2.17.0, )", - "resolved": "2.17.0", - "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", + "requested": "[2.14.0, )", + "resolved": "2.14.0", + "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "5.0.0" + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory": "4.5.4" } }, "Microsoft.AspNetCore.Http": { @@ -96,9 +97,9 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[13.0.1, )", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + "requested": "[11.0.2, )", + "resolved": "11.0.2", + "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" }, "System.Runtime.Caching": { "type": "Direct", @@ -716,11 +717,10 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==", + "resolved": "4.6.0", + "contentHash": "mbBgoR0rRfl2uimsZ2avZY8g7Xnh1Mza0rJZLPcxqiMWlkGukjmRkuMJ/er+AhQuiRIh80CR/Hpeztr80seV5g==", "dependencies": { - "System.Memory": "4.5.4", - "System.Runtime.CompilerServices.Unsafe": "5.0.0" + "System.Memory": "4.5.3" } }, "System.Diagnostics.Process": { diff --git a/test/packages.lock.json b/test/packages.lock.json index 898fc594c..bdef10366 100644 --- a/test/packages.lock.json +++ b/test/packages.lock.json @@ -17,16 +17,16 @@ }, "Microsoft.NET.Sdk.Functions": { "type": "Direct", - "requested": "[4.1.3, )", - "resolved": "4.1.3", - "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", + "requested": "[4.1.1, )", + "resolved": "4.1.1", + "contentHash": "G+b+bHtta7P8KPItSWygbVwQhwamU/WWmBoiv3snf8ScVYai3PbC1JSW3H22X+askqxhiw/Tx0yZKdE5oKMhRQ==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", - "Newtonsoft.Json": "13.0.1" + "Newtonsoft.Json": "11.0.2" } }, "Microsoft.NET.Test.Sdk": { @@ -51,9 +51,9 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[13.0.1, )", - "resolved": "13.0.1", - "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + "requested": "[11.0.2, )", + "resolved": "11.0.2", + "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" }, "xunit": { "type": "Direct", @@ -149,8 +149,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.8", - "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", + "resolved": "5.2.4", + "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -158,59 +158,59 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", + "resolved": "2.1.0", + "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", + "resolved": "2.1.0", + "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http": "2.2.0", - "Microsoft.AspNetCore.Http.Extensions": "2.2.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Http.Extensions": "2.1.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", + "resolved": "2.1.0", + "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", + "resolved": "2.1.0", + "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Authorization": "2.2.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Authorization": "2.1.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", + "resolved": "2.1.0", + "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", + "resolved": "2.1.0", + "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.2.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Http.Features": "2.1.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { @@ -224,12 +224,12 @@ }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", + "resolved": "2.1.0", + "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Net.Http.Headers": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Net.Http.Headers": "2.1.0", "System.Buffers": "4.5.0" } }, @@ -243,8 +243,8 @@ }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", + "resolved": "2.1.0", + "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -252,81 +252,80 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", + "resolved": "2.1.0", + "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Net.Http.Headers": "2.2.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", + "Microsoft.Net.Http.Headers": "2.1.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.2.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Http": "2.2.0", - "Microsoft.AspNetCore.Http.Extensions": "2.2.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", - "Microsoft.AspNetCore.Routing": "2.2.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Extensions.DependencyInjection": "2.2.0", + "resolved": "2.1.0", + "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.1.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Http.Extensions": "2.1.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", + "Microsoft.AspNetCore.Routing": "2.1.0", + "Microsoft.Extensions.DependencyInjection": "2.1.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.1" + "System.Threading.Tasks.Extensions": "4.5.0" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", + "resolved": "2.1.0", + "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.2.0", - "Microsoft.AspNetCore.Mvc.Core": "2.2.0" + "Microsoft.AspNetCore.JsonPatch": "2.1.0", + "Microsoft.AspNetCore.Mvc.Core": "2.1.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", + "resolved": "2.1.0", + "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.6", - "Microsoft.AspNetCore.Mvc.Core": "2.2.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", - "Microsoft.AspNetCore.WebUtilities": "2.2.0" + "Microsoft.AspNet.WebApi.Client": "5.2.4", + "Microsoft.AspNetCore.Mvc.Core": "2.1.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", + "Microsoft.AspNetCore.WebUtilities": "2.1.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", + "resolved": "2.1.0", + "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.2.2", - "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", + "resolved": "2.1.0", + "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.2.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0", - "Microsoft.Extensions.ObjectPool": "2.2.0", - "Microsoft.Extensions.Options": "2.2.0" + "Microsoft.AspNetCore.Http.Extensions": "2.1.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.ObjectPool": "2.1.0", + "Microsoft.Extensions.Options": "2.1.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", + "resolved": "2.1.0", + "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" } }, "Microsoft.AspNetCore.WebUtilities": { @@ -364,15 +363,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.2.0", - "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", + "resolved": "3.0.2", + "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.8", - "Microsoft.AspNetCore.Http": "2.2.2", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", - "Microsoft.AspNetCore.Routing": "2.2.2", - "Microsoft.Azure.WebJobs": "3.0.32" + "Microsoft.AspNet.WebApi.Client": "5.2.4", + "Microsoft.AspNetCore.Http": "2.1.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", + "Microsoft.AspNetCore.Routing": "2.1.0", + "Microsoft.Azure.WebJobs": "3.0.2" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -464,10 +463,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", + "resolved": "2.1.1", + "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.1" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -507,10 +506,10 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", + "resolved": "2.1.0", + "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { @@ -532,10 +531,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", + "resolved": "2.1.0", + "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.2.0" + "Microsoft.Extensions.Primitives": "2.1.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -566,13 +565,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", + "resolved": "2.1.0", + "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", - "Microsoft.Extensions.Logging.Abstractions": "2.2.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", + "Microsoft.Extensions.Logging.Abstractions": "2.1.0" } }, "Microsoft.Extensions.Logging": { @@ -588,8 +587,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.2.0", - "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" + "resolved": "2.1.1", + "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -1945,11 +1944,11 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.17.0, )", + "Microsoft.ApplicationInsights": "[2.14.0, )", "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs": "[3.0.32, )", "Microsoft.Data.SqlClient": "[5.0.1, )", - "Newtonsoft.Json": "[13.0.1, )", + "Newtonsoft.Json": "[11.0.2, )", "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } @@ -1960,17 +1959,18 @@ "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )", - "Microsoft.NET.Sdk.Functions": "[4.1.3, )", - "Newtonsoft.Json": "[13.0.1, )" + "Microsoft.NET.Sdk.Functions": "[4.1.1, )", + "Newtonsoft.Json": "[11.0.2, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.17.0, )", - "resolved": "2.17.0", - "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", + "requested": "[2.14.0, )", + "resolved": "2.14.0", + "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "5.0.0" + "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Memory": "4.5.4" } }, "Microsoft.Azure.WebJobs": { From 02d5da795b03fc4d1d498546b3bf568d99d3f792 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 28 Oct 2022 11:13:42 -0700 Subject: [PATCH 56/77] undo unintended changes --- performance/SqlBindingBenchmarks.cs | 4 ++-- performance/SqlTriggerBindingPerformance.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/performance/SqlBindingBenchmarks.cs b/performance/SqlBindingBenchmarks.cs index 77e4e7b50..7917e17e2 100644 --- a/performance/SqlBindingBenchmarks.cs +++ b/performance/SqlBindingBenchmarks.cs @@ -9,8 +9,8 @@ public class SqlBindingPerformance { public static void Main() { - //BenchmarkRunner.Run(); - //BenchmarkRunner.Run(); + BenchmarkRunner.Run(); + BenchmarkRunner.Run(); BenchmarkRunner.Run(); } } diff --git a/performance/SqlTriggerBindingPerformance.cs b/performance/SqlTriggerBindingPerformance.cs index d96c3a115..43a63f5ea 100644 --- a/performance/SqlTriggerBindingPerformance.cs +++ b/performance/SqlTriggerBindingPerformance.cs @@ -22,7 +22,7 @@ public void GlobalSetup() [Arguments(1)] [Arguments(10)] [Arguments(100)] - // [Arguments(1000)] + [Arguments(1000)] public async Task ProductsTriggerTest(int count) { await this.WaitForProductChanges( From 55903a3ea73078b22978de2d9295211b8ad2d1ed Mon Sep 17 00:00:00 2001 From: AmeyaRele <35621237+AmeyaRele@users.noreply.github.com> Date: Wed, 2 Nov 2022 13:29:38 +0530 Subject: [PATCH 57/77] Add trigger scaling configuration (#428) * Add trigger scaling config * Address PR comments * Fix broken tests * Add unit tests * Address review comments * Update unit tests * Make variable names consistent * Add logging Co-authored-by: Jatin Sanghvi <20547963+JatinSanghvi@users.noreply.github.com> --- README.md | 5 + src/Telemetry/Telemetry.cs | 1 + src/TriggerBinding/SqlTableChangeMonitor.cs | 1 + src/TriggerBinding/SqlTriggerConstants.cs | 1 + src/TriggerBinding/SqlTriggerListener.cs | 33 ++++-- .../TriggerBinding/SqlTriggerListenerTests.cs | 103 +++++++++++++++--- 6 files changed, 118 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index c81ab170f..9e2d7375a 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Azure SQL bindings for Azure Functions are supported for: - [Trigger Binding Configuration](#trigger-binding-configuration) - [Sql_Trigger_BatchSize](#sql_trigger_batchsize) - [Sql_Trigger_PollingIntervalMs](#sql_trigger_pollingintervalms) + - [Sql_Trigger_MaxChangesPerWorker](#sql_trigger_maxchangesperworker) - [More Samples](#more-samples) - [Input Binding](#input-binding) - [Query String](#query-string) @@ -578,6 +579,10 @@ This controls the number of changes processed at once before being sent to the t This controls the delay in milliseconds between processing each batch of changes. +#### Sql_Trigger_MaxChangesPerWorker + +This controls the upper limit on the number of pending changes in the user table that are allowed per application-worker. If the count of changes exceeds this limit, it may result in a scale out. The setting only applies for Azure Function Apps with runtime driven scaling enabled. See the [Scaling](#scaling) section for more information. + ## More Samples ### Input Binding diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index cc0aa9313..91dd340c9 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -417,6 +417,7 @@ public enum TelemetryErrorName GetPrimaryKeys, GetScaleStatus, GetUnprocessedChangeCount, + InvalidConfigurationValue, MissingPrimaryKeys, NoPrimaryKeys, ProcessChanges, diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 28353e4d3..79a1944e2 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -98,6 +98,7 @@ internal sealed class SqlTableChangeMonitor : IDisposable /// List of primary key column names in the user table /// Defines contract for triggering user function /// Facilitates logging of messages + /// Provides configuration values /// Properties passed in telemetry events public SqlTableChangeMonitor( string connectionString, diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index f79c617fa..533438500 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -28,5 +28,6 @@ internal static class SqlTriggerConstants public const string ConfigKey_SqlTrigger_BatchSize = "Sql_Trigger_BatchSize"; public const string ConfigKey_SqlTrigger_PollingInterval = "Sql_Trigger_PollingIntervalMs"; + public const string ConfigKey_SqlTrigger_MaxChangesPerWorker = "Sql_Trigger_MaxChangesPerWorker"; } } \ No newline at end of file diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 359832f91..187a00383 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -32,6 +32,10 @@ internal sealed class SqlTriggerListener : IListener, IScaleMonitor : IListener, IScaleMonitor _telemetryProps = new Dictionary(); + private readonly int _maxChangesPerWorker; private SqlTableChangeMonitor _changeMonitor; private int _listenerState = ListenerNotStarted; @@ -64,10 +69,24 @@ public SqlTriggerListener(string connectionString, string tableName, string user this._executor = executor ?? throw new ArgumentNullException(nameof(executor)); this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - + int configuredMaxChangesPerWorker; // Do not convert the scale-monitor ID to lower-case string since SQL table names can be case-sensitive // depending on the collation of the current database. this._scaleMonitorDescriptor = new ScaleMonitorDescriptor($"{userFunctionId}-SqlTrigger-{tableName}"); + + // In case converting from string to int is not possible from the user input. + try + { + configuredMaxChangesPerWorker = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_MaxChangesPerWorker); + } + catch (Exception ex) + { + this._logger.LogError($"Failed to resolve integer value from user configured setting '{SqlTriggerConstants.ConfigKey_SqlTrigger_MaxChangesPerWorker}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + TelemetryInstance.TrackException(TelemetryErrorName.InvalidConfigurationValue, ex, this._telemetryProps); + + configuredMaxChangesPerWorker = DefaultMaxChangesPerWorker; + } + this._maxChangesPerWorker = configuredMaxChangesPerWorker > 0 ? configuredMaxChangesPerWorker : DefaultMaxChangesPerWorker; } public void Cancel() @@ -543,10 +562,6 @@ private ScaleStatus GetScaleStatusCore(int workerCount, SqlTriggerMetrics[] metr // certain reliability. These samples roughly cover the timespan of past 40 seconds. const int minSamplesForScaling = 5; - // NOTE: please ensure the Readme file and other public documentation are also updated if this value ever - // needs to be changed. - const int maxChangesPerWorker = 1000; - var status = new ScaleStatus { Vote = ScaleVote.None, @@ -563,11 +578,11 @@ private ScaleStatus GetScaleStatusCore(int workerCount, SqlTriggerMetrics[] metr metrics = metrics.TakeLast(minSamplesForScaling).ToArray(); string counts = string.Join(", ", metrics.Select(metric => metric.UnprocessedChangeCount)); - this._logger.LogInformation($"Unprocessed change counts: [{counts}], worker count: {workerCount}, maximum changes per worker: {maxChangesPerWorker}."); + this._logger.LogInformation($"Unprocessed change counts: [{counts}], worker count: {workerCount}, maximum changes per worker: {this._maxChangesPerWorker}."); // Add worker if the count of unprocessed changes per worker exceeds the maximum limit. long lastUnprocessedChangeCount = metrics.Last().UnprocessedChangeCount; - if (lastUnprocessedChangeCount > workerCount * maxChangesPerWorker) + if (lastUnprocessedChangeCount > workerCount * this._maxChangesPerWorker) { status.Vote = ScaleVote.ScaleOut; this._logger.LogInformation($"Requesting scale-out: Found too many unprocessed changes for table: '{this._userTable.FullName}' relative to the number of workers."); @@ -590,7 +605,7 @@ private ScaleStatus GetScaleStatusCore(int workerCount, SqlTriggerMetrics[] metr SqlTriggerMetrics referenceMetric = metrics.First(metric => metric.Timestamp > referenceTime); long expectedUnprocessedChangeCount = (2 * metrics[metrics.Length - 1].UnprocessedChangeCount) - referenceMetric.UnprocessedChangeCount; - if (expectedUnprocessedChangeCount > workerCount * maxChangesPerWorker) + if (expectedUnprocessedChangeCount > workerCount * this._maxChangesPerWorker) { status.Vote = ScaleVote.ScaleOut; this._logger.LogInformation($"Requesting scale-out: Found the unprocessed changes for table: '{this._userTable.FullName}' to be continuously increasing" + @@ -607,7 +622,7 @@ private ScaleStatus GetScaleStatusCore(int workerCount, SqlTriggerMetrics[] metr if (isDecreasing) { // Scale in only if the count of unprocessed changes will not exceed the combined limit post the scale-in operation. - if (lastUnprocessedChangeCount <= (workerCount - 1) * maxChangesPerWorker) + if (lastUnprocessedChangeCount <= (workerCount - 1) * this._maxChangesPerWorker) { status.Vote = ScaleVote.ScaleIn; this._logger.LogInformation($"Requesting scale-in: Found table: '{this._userTable.FullName}' to be either idle or the unprocessed changes to be continuously decreasing."); diff --git a/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs b/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs index 28319363b..8995bc18d 100644 --- a/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs +++ b/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs @@ -201,33 +201,70 @@ public void ScaleMonitorGetScaleStatus_CountNotIncreasingOrDecreasing_ReturnsNon Assert.Contains("Requesting no-scaling: Found the number of unprocessed changes for table: 'testTableName' to not require scaling.", logMessages); } + [Theory] + [InlineData("1")] + [InlineData("100")] + [InlineData("10000")] + public void ScaleMonitorGetScaleStatus_UserConfiguredMaxChangesPerWorker_RespectsConfiguration(string maxChangesPerWorker) + { + (IScaleMonitor monitor, _) = GetScaleMonitor(maxChangesPerWorker); + + ScaleStatusContext context; + ScaleStatus scaleStatus; + + int max = int.Parse(maxChangesPerWorker); + + context = GetScaleStatusContext(new int[] { 0, 0, 0, 0, 10 * max }, 10); + scaleStatus = monitor.GetScaleStatus(context); + Assert.Equal(ScaleVote.None, scaleStatus.Vote); + + context = GetScaleStatusContext(new int[] { 0, 0, 0, 0, (10 * max) + 1 }, 10); + scaleStatus = monitor.GetScaleStatus(context); + Assert.Equal(ScaleVote.ScaleOut, scaleStatus.Vote); + + context = GetScaleStatusContext(new int[] { (9 * max) + 4, (9 * max) + 3, (9 * max) + 2, (9 * max) + 1, 9 * max }, 10); + scaleStatus = monitor.GetScaleStatus(context); + Assert.Equal(ScaleVote.ScaleIn, scaleStatus.Vote); + } + + [Theory] + [InlineData("invalidValue")] + [InlineData("-1")] + [InlineData("0")] + [InlineData("10000000000")] + public void ScaleMonitorGetScaleStatus_InvalidUserConfiguredMaxChangesPerWorker_UsesDefaultValue(string maxChangesPerWorker) + { + (IScaleMonitor monitor, _) = GetScaleMonitor(maxChangesPerWorker); + + ScaleStatusContext context; + ScaleStatus scaleStatus; + + context = GetScaleStatusContext(new int[] { 0, 0, 0, 0, 10000 }, 10); + scaleStatus = monitor.GetScaleStatus(context); + Assert.Equal(ScaleVote.None, scaleStatus.Vote); + + context = GetScaleStatusContext(new int[] { 0, 0, 0, 0, 10001 }, 10); + scaleStatus = monitor.GetScaleStatus(context); + Assert.Equal(ScaleVote.ScaleOut, scaleStatus.Vote); + } + private static IScaleMonitor GetScaleMonitor(string tableName, string userFunctionId) { + Mock mockConfiguration = CreateMockConfiguration(); + return new SqlTriggerListener( "testConnectionString", tableName, userFunctionId, Mock.Of(), Mock.Of(), - Mock.Of()); + mockConfiguration.Object); } - private static (IScaleMonitor monitor, List logMessages) GetScaleMonitor() + private static (IScaleMonitor monitor, List logMessages) GetScaleMonitor(string maxChangesPerWorker = null) { - // Since multiple threads are not involved when computing the scale-status, it should be okay to not use - // a thread-safe collection for storing the log messages. - var logMessages = new List(); - var mockLogger = new Mock(); - - // Both LogInformation() and LogDebug() are extension methods. Since the extension methods are static, they - // cannot be mocked. Hence, we need to setup callback on an inner class method that gets eventually called - // by these methods in order to extract the log message. - mockLogger - .Setup(logger => logger.Log(It.IsAny(), 0, It.IsAny(), null, It.IsAny>())) - .Callback((LogLevel logLevel, EventId eventId, object state, Exception exception, Func formatter) => - { - logMessages.Add(state.ToString()); - }); + (Mock mockLogger, List logMessages) = CreateMockLogger(); + Mock mockConfiguration = CreateMockConfiguration(maxChangesPerWorker); IScaleMonitor monitor = new SqlTriggerListener( "testConnectionString", @@ -235,7 +272,7 @@ private static (IScaleMonitor monitor, List logMessag "testUserFunctionId", Mock.Of(), mockLogger.Object, - Mock.Of()); + mockConfiguration.Object); return (monitor, logMessages); } @@ -256,5 +293,37 @@ private static ScaleStatusContext GetScaleStatusContext(int[] unprocessedChangeC WorkerCount = workerCount, }; } + + private static (Mock logger, List logMessages) CreateMockLogger() + { + // Since multiple threads are not involved when computing the scale-status, it should be okay to not use + // a thread-safe collection for storing the log messages. + var logMessages = new List(); + var mockLogger = new Mock(); + + // Both LogInformation and LogDebug are extension (static) methods and cannot be mocked. Hence, we need to + // setup callback on an inner class method that gets eventually called by these methods in order to extract + // the log message. + mockLogger + .Setup(logger => logger.Log(It.IsAny(), 0, It.IsAny(), null, It.IsAny>())) + .Callback((LogLevel logLevel, EventId eventId, object state, Exception exception, Func formatter) => + { + logMessages.Add(state.ToString()); + }); + + return (mockLogger, logMessages); + } + + private static Mock CreateMockConfiguration(string maxChangesPerWorker = null) + { + // GetValue is an extension (static) method and cannot be mocked. However, it calls GetSection which + // expects us to return IConfigurationSection, which is why GetSection is mocked. + var mockConfiguration = new Mock(); + mockConfiguration + .Setup(x => x.GetSection("Sql_Trigger_MaxChangesPerWorker")) + .Returns(Mock.Of(section => section.Value == maxChangesPerWorker)); + + return mockConfiguration; + } } } \ No newline at end of file From be7c63b57b281fb54383143120aec0f61096643a Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Wed, 2 Nov 2022 10:02:27 -0700 Subject: [PATCH 58/77] Perf test fixes --- performance/.editorconfig | 9 ++++ performance/SqlTriggerBindingPerformance.cs | 18 +------ .../SqlTriggerBindingPerformanceTestBase.cs | 48 +++++++++++++++++++ test/Integration/IntegrationTestBase.cs | 33 ++++++++----- .../SqlTriggerBindingIntegrationTests.cs | 21 ++++---- test/README.md | 20 +++++++- 6 files changed, 109 insertions(+), 40 deletions(-) create mode 100644 performance/.editorconfig create mode 100644 performance/SqlTriggerBindingPerformanceTestBase.cs diff --git a/performance/.editorconfig b/performance/.editorconfig new file mode 100644 index 000000000..0082aa2a0 --- /dev/null +++ b/performance/.editorconfig @@ -0,0 +1,9 @@ +# See https://github.com/dotnet/roslyn-analyzers/blob/main/.editorconfig for an example on different settings and how they're used + +[*.cs] + +# Disabled +dotnet_diagnostic.CA1309.severity = silent # Use ordinal StringComparison - this isn't important for tests and just adds clutter +dotnet_diagnostic.CA1305.severity = silent # Specify IFormatProvider - this isn't important for tests and just adds clutter +dotnet_diagnostic.CA1707.severity = silent # Identifiers should not contain underscores - this helps make test names more readable +dotnet_diagnostic.CA2201.severity = silent # Do not raise reserved exception types - tests can throw whatever they want \ No newline at end of file diff --git a/performance/SqlTriggerBindingPerformance.cs b/performance/SqlTriggerBindingPerformance.cs index 43a63f5ea..233339f43 100644 --- a/performance/SqlTriggerBindingPerformance.cs +++ b/performance/SqlTriggerBindingPerformance.cs @@ -4,17 +4,16 @@ using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; -using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration; using BenchmarkDotNet.Attributes; namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance { - public class SqlTriggerBindingPerformance : SqlTriggerBindingIntegrationTests + [MemoryDiagnoser] + public class SqlTriggerBindingPerformance : SqlTriggerBindingPerformanceTestBase { [GlobalSetup] public void GlobalSetup() { - this.EnableChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); } @@ -34,18 +33,5 @@ await this.WaitForProductChanges( id => id * 100, this.GetBatchProcessingTimeout(1, count)); } - - [IterationCleanup] - public void IterationCleanup() - { - // Delete all rows in Products table after each iteration - this.ExecuteNonQuery("TRUNCATE TABLE Products"); - } - - [GlobalCleanup] - public void GlobalCleanup() - { - this.Dispose(); - } } } \ No newline at end of file diff --git a/performance/SqlTriggerBindingPerformanceTestBase.cs b/performance/SqlTriggerBindingPerformanceTestBase.cs new file mode 100644 index 000000000..d0f078624 --- /dev/null +++ b/performance/SqlTriggerBindingPerformanceTestBase.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance +{ + public class SqlTriggerBindingPerformanceTestBase : SqlTriggerBindingIntegrationTests + { + [IterationSetup] + public void IterationSetup() + { + this.SetChangeTrackingForTable("Products", true); + } + + [IterationCleanup] + public void IterationCleanup() + { + this.DisposeFunctionHosts(); + this.SetChangeTrackingForTable("Products", false); + // Delete all rows in Products table after each iteration so we start fresh each time + this.ExecuteNonQuery("DELETE FROM Products"); + // Delete the leases table, otherwise we may end up getting blocked by leases from a previous run + this.ExecuteNonQuery(@"DECLARE @cmd varchar(100) +DECLARE cmds CURSOR FOR +SELECT 'DROP TABLE az_func.' + Name + '' +FROM sys.tables +WHERE Name LIKE 'Leases_%' + +OPEN cmds +WHILE 1 = 1 +BEGIN + FETCH cmds INTO @cmd + IF @@fetch_status != 0 BREAK + EXEC(@cmd) +END +CLOSE cmds; +DEALLOCATE cmds"); + } + + [GlobalCleanup] + public void GlobalCleanup() + { + this.Dispose(); + } + } +} \ No newline at end of file diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index 5d6a80a8e..fe06b968d 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -394,18 +394,7 @@ public void Dispose() this.LogOutput($"Failed to close connection. Error: {e1.Message}"); } - foreach (Process functionHost in this.FunctionHostList) - { - try - { - functionHost.Kill(); - functionHost.Dispose(); - } - catch (Exception e2) - { - this.LogOutput($"Failed to stop function host, Error: {e2.Message}"); - } - } + this.DisposeFunctionHosts(); try { @@ -432,6 +421,26 @@ public void Dispose() GC.SuppressFinalize(this); } + /// + /// Disposes all the running function hosts + /// + protected void DisposeFunctionHosts() + { + foreach (Process functionHost in this.FunctionHostList) + { + try + { + functionHost.Kill(); + functionHost.Dispose(); + } + catch (Exception e2) + { + this.LogOutput($"Failed to stop function host, Error: {e2.Message}"); + } + } + this.FunctionHostList.Clear(); + } + protected async Task SendInputRequest(string functionName, string query = "") { string requestUri = $"http://localhost:{this.Port}/api/{functionName}/{query}"; diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index a1f6d4c80..738c9f3c7 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -29,7 +29,7 @@ public SqlTriggerBindingIntegrationTests(ITestOutputHelper output = null) : base [Fact] public async Task SingleOperationTriggerTest() { - this.EnableChangeTrackingForTable("Products"); + this.SetChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); int firstId = 1; @@ -82,7 +82,7 @@ public async Task BatchSizeOverrideTriggerTest() const int batchSize = SqlTableChangeMonitor.DefaultBatchSize * 4; const int firstId = 1; const int lastId = batchSize; - this.EnableChangeTrackingForTable("Products"); + this.SetChangeTrackingForTable("Products"); var taskCompletionSource = new TaskCompletionSource(); DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( taskCompletionSource, @@ -120,10 +120,10 @@ public async Task PollingIntervalOverrideTriggerTest() const int firstId = 1; // Use enough items to require 5 batches to be processed - the test will // only wait for the expected time and timeout if the default polling - // interval isn't actually modified. + // interval isn't actually modified. const int lastId = SqlTableChangeMonitor.DefaultBatchSize * 5; const int pollingIntervalMs = SqlTableChangeMonitor.DefaultPollingIntervalMs / 2; - this.EnableChangeTrackingForTable("Products"); + this.SetChangeTrackingForTable("Products"); var taskCompletionSource = new TaskCompletionSource(); DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( taskCompletionSource, @@ -160,7 +160,7 @@ public async Task MultiOperationTriggerTest() { int firstId = 1; int lastId = 5; - this.EnableChangeTrackingForTable("Products"); + this.SetChangeTrackingForTable("Products"); this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); // 1. Insert + multiple updates to a row are treated as single insert with latest row values. @@ -241,7 +241,7 @@ public async Task MultiFunctionTriggerTest() const string Trigger1Changes = "Trigger1 Changes: "; const string Trigger2Changes = "Trigger2 Changes: "; - this.EnableChangeTrackingForTable("Products"); + this.SetChangeTrackingForTable("Products"); string functionList = $"{nameof(MultiFunctionTrigger.MultiFunctionTrigger1)} {nameof(MultiFunctionTrigger.MultiFunctionTrigger2)}"; this.StartFunctionHost(functionList, SupportedLanguages.CSharp, useTestFolder: true); @@ -363,7 +363,7 @@ public async Task MultiFunctionTriggerTest() [Fact] public async Task MultiHostTriggerTest() { - this.EnableChangeTrackingForTable("Products"); + this.SetChangeTrackingForTable("Products"); // Prepare three function host processes. this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); @@ -479,11 +479,11 @@ ALTER DATABASE [{this.DatabaseName}] "); } - protected void EnableChangeTrackingForTable(string tableName) + protected void SetChangeTrackingForTable(string tableName, bool enable = true) { this.ExecuteNonQuery($@" ALTER TABLE [dbo].[{tableName}] - ENABLE CHANGE_TRACKING; + {(enable ? "ENABLE" : "DISABLE")} CHANGE_TRACKING; "); } @@ -559,6 +559,7 @@ void MonitorOutputData(object sender, DataReceivedEventArgs e) await actions(); // Now wait until either we timeout or we've gotten all the expected changes, whichever comes first + Console.WriteLine("Waiting for product changes"); await taskCompletion.Task.TimeoutAfter(TimeSpan.FromMilliseconds(timeoutMs), $"Timed out waiting for {operation} changes."); // Unhook handler since we're done monitoring these changes so we aren't checking other changes done later @@ -610,7 +611,7 @@ void OutputHandler(object sender, DataReceivedEventArgs e) /// /// Gets a timeout value to use when processing the given number of changes, based on the - /// default batch size and polling interval. + /// default batch size and polling interval. /// /// The first ID in the batch to process /// The last ID in the batch to process diff --git a/test/README.md b/test/README.md index 7be921e56..42095602d 100644 --- a/test/README.md +++ b/test/README.md @@ -56,7 +56,7 @@ Our integration tests are based on functions from the samples project. To run in this.StartFunctionHost(nameof(), lang); // Replace with the class name of the function this test is running against // test code here } - ``` + ``` Ex: When the test has parameters: ``` @@ -86,4 +86,20 @@ Our integration tests are based on functions from the samples project. To run in this.StartFunctionHost(nameof(), lang); // Replace with the class name of the function this test is running against // test code here } - ``` \ No newline at end of file + ``` + +## Troubleshooting Tests + +This section lists some things to try to help troubleshoot test failures + +### Enable debug logging on the Function + +Enabling debug logging can greatly increase the information available which can help track down issues or understand at least where the problem may be. To enable debug logging for the Function open [host.json](../samples/samples-csharp/host.json) and add the following property to the `logLevel` section, then rebuild and re-run your test. + +```json +"logLevel": { + "default": "Debug" +} +``` + +WARNING : Doing this will add a not-insignificant overhead to the test run duration from writing all the additional content to the log files, which may cause timeouts to occur in tests. If this happens you can temporarily increase those timeouts while debug logging is enabled to avoid having unexpected failures. \ No newline at end of file From 181a77e7044341a2f59d97382669b4217481e937 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Wed, 2 Nov 2022 10:05:38 -0700 Subject: [PATCH 59/77] truncate --- performance/SqlTriggerBindingPerformanceTestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/performance/SqlTriggerBindingPerformanceTestBase.cs b/performance/SqlTriggerBindingPerformanceTestBase.cs index d0f078624..d891118a0 100644 --- a/performance/SqlTriggerBindingPerformanceTestBase.cs +++ b/performance/SqlTriggerBindingPerformanceTestBase.cs @@ -20,7 +20,7 @@ public void IterationCleanup() this.DisposeFunctionHosts(); this.SetChangeTrackingForTable("Products", false); // Delete all rows in Products table after each iteration so we start fresh each time - this.ExecuteNonQuery("DELETE FROM Products"); + this.ExecuteNonQuery("TRUNCATE TABLE Products"); // Delete the leases table, otherwise we may end up getting blocked by leases from a previous run this.ExecuteNonQuery(@"DECLARE @cmd varchar(100) DECLARE cmds CURSOR FOR From 69d96345585d258c5a2b430411646a57d92c5694 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Wed, 2 Nov 2022 13:50:11 -0700 Subject: [PATCH 60/77] e2 -> ex --- test/Integration/IntegrationTestBase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index fe06b968d..934db3bdc 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -433,9 +433,9 @@ protected void DisposeFunctionHosts() functionHost.Kill(); functionHost.Dispose(); } - catch (Exception e2) + catch (Exception ex) { - this.LogOutput($"Failed to stop function host, Error: {e2.Message}"); + this.LogOutput($"Failed to stop function host, Error: {ex.Message}"); } } this.FunctionHostList.Clear(); From 6a7ba22e694aa6d36a7e644e1a6547909950dde2 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Wed, 2 Nov 2022 16:30:59 -0700 Subject: [PATCH 61/77] More perf testing fixes/improvements --- .../SqlTriggerBindingIntegrationTests.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 738c9f3c7..339c95856 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Text; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common; using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; @@ -489,10 +490,15 @@ ALTER TABLE [dbo].[{tableName}] protected void InsertProducts(int firstId, int lastId) { - int count = lastId - firstId + 1; - this.ExecuteNonQuery( - "INSERT INTO [dbo].[Products] VALUES\n" + - string.Join(",\n", Enumerable.Range(firstId, count).Select(id => $"({id}, 'Product {id}', {id * 100})")) + ";"); + // Only 1000 items are allowed to be inserted into a single INSERT statement so if we have more than 1000 batch them up into separate statements + var builder = new StringBuilder(); + do + { + int batchCount = Math.Min(lastId - firstId + 1, 1000); + builder.Append($"INSERT INTO [dbo].[Products] VALUES {string.Join(",\n", Enumerable.Range(firstId, batchCount).Select(id => $"({id}, 'Product {id}', {id * 100})"))}; "); + firstId += batchCount; + } while (firstId < lastId); + this.ExecuteNonQuery(builder.ToString()); } protected void UpdateProducts(int firstId, int lastId) @@ -559,7 +565,7 @@ void MonitorOutputData(object sender, DataReceivedEventArgs e) await actions(); // Now wait until either we timeout or we've gotten all the expected changes, whichever comes first - Console.WriteLine("Waiting for product changes"); + Console.WriteLine($"[{DateTime.UtcNow:u}] Waiting for product changes ({timeoutMs}ms)"); await taskCompletion.Task.TimeoutAfter(TimeSpan.FromMilliseconds(timeoutMs), $"Timed out waiting for {operation} changes."); // Unhook handler since we're done monitoring these changes so we aren't checking other changes done later From 238ce8337fe9d76a50ab07eacf7667f956673087 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Wed, 2 Nov 2022 16:34:19 -0700 Subject: [PATCH 62/77] Add operation --- test/Integration/SqlTriggerBindingIntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 339c95856..85e6bc954 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -565,7 +565,7 @@ void MonitorOutputData(object sender, DataReceivedEventArgs e) await actions(); // Now wait until either we timeout or we've gotten all the expected changes, whichever comes first - Console.WriteLine($"[{DateTime.UtcNow:u}] Waiting for product changes ({timeoutMs}ms)"); + Console.WriteLine($"[{DateTime.UtcNow:u}] Waiting for {operation} changes ({timeoutMs}ms)"); await taskCompletion.Task.TimeoutAfter(TimeSpan.FromMilliseconds(timeoutMs), $"Timed out waiting for {operation} changes."); // Unhook handler since we're done monitoring these changes so we aren't checking other changes done later From 2bd0bed86fa5d7f289e2e7c290c473d581104ef5 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Thu, 3 Nov 2022 10:24:49 -0700 Subject: [PATCH 63/77] Static import SqlTriggerConstants --- src/TriggerBinding/SqlTableChangeMonitor.cs | 84 +++++++++++---------- src/TriggerBinding/SqlTriggerListener.cs | 27 +++---- 2 files changed, 58 insertions(+), 53 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 79a1944e2..eb67ef52f 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -12,6 +12,7 @@ using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry; using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry; +using static Microsoft.Azure.WebJobs.Extensions.Sql.SqlTriggerConstants; using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; @@ -126,8 +127,8 @@ public SqlTableChangeMonitor( this._telemetryProps = telemetryProps ?? new Dictionary(); // Check if there's config settings to override the default batch size/polling interval values - int? configuredBatchSize = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize); - int? configuredPollingInterval = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_PollingInterval); + int? configuredBatchSize = configuration.GetValue(ConfigKey_SqlTrigger_BatchSize); + int? configuredPollingInterval = configuration.GetValue(ConfigKey_SqlTrigger_PollingInterval); this._batchSize = configuredBatchSize ?? this._batchSize; this._pollingIntervalInMs = configuredPollingInterval ?? this._pollingIntervalInMs; var monitorStartProps = new Dictionary(telemetryProps) @@ -646,7 +647,7 @@ private long RecomputeLastSyncVersion() var changeVersionSet = new SortedSet(); foreach (IReadOnlyDictionary row in this._rows) { - string changeVersion = row[SqlTriggerConstants.SysChangeVersionColumnName].ToString(); + string changeVersion = row[SysChangeVersionColumnName].ToString(); changeVersionSet.Add(long.Parse(changeVersion, CultureInfo.InvariantCulture)); } @@ -722,11 +723,11 @@ private SqlCommand BuildUpdateTablesPreInvocation(SqlConnection connection, SqlT DECLARE @last_sync_version bigint; SELECT @last_sync_version = LastSyncVersion - FROM {SqlTriggerConstants.GlobalStateTableName} + FROM {GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; IF @last_sync_version < @min_valid_version - UPDATE {SqlTriggerConstants.GlobalStateTableName} + UPDATE {GlobalStateTableName} SET LastSyncVersion = @min_valid_version WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; "; @@ -749,23 +750,26 @@ private SqlCommand BuildGetChangesCommand(SqlConnection connection, SqlTransacti string getChangesQuery = $@" DECLARE @last_sync_version bigint; SELECT @last_sync_version = LastSyncVersion - FROM {SqlTriggerConstants.GlobalStateTableName} + FROM {GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; SELECT TOP {this._batchSize} {selectList}, - c.{SqlTriggerConstants.SysChangeVersionColumnName}, c.SYS_CHANGE_OPERATION, - l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName}, l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName}, l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} + c.{SysChangeVersionColumnName}, + c.SYS_CHANGE_OPERATION, + l.{LeasesTableChangeVersionColumnName}, + l.{LeasesTableAttemptCountColumnName}, + l.{LeasesTableLeaseExpirationTimeColumnName} FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCondition} WHERE - (l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NULL AND - (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.{SqlTriggerConstants.SysChangeVersionColumnName}) OR - l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND - (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}) - ORDER BY c.{SqlTriggerConstants.SysChangeVersionColumnName} ASC; - "; + (l.{LeasesTableLeaseExpirationTimeColumnName} IS NULL AND + (l.{LeasesTableChangeVersionColumnName} IS NULL OR l.{LeasesTableChangeVersionColumnName} < c.{SysChangeVersionColumnName}) OR + l.{LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME() + ) AND + (l.{LeasesTableAttemptCountColumnName} IS NULL OR l.{LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}) + ORDER BY c.{SysChangeVersionColumnName} ASC;"; return new SqlCommand(getChangesQuery, connection, transaction); } @@ -783,17 +787,17 @@ private SqlCommand BuildGetUnprocessedChangesCommand(SqlConnection connection) string getUnprocessedChangesQuery = $@" DECLARE @last_sync_version bigint; SELECT @last_sync_version = LastSyncVersion - FROM {SqlTriggerConstants.GlobalStateTableName} + FROM {GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; SELECT COUNT_BIG(*) FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} WHERE - (l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NULL AND - (l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} < c.{SqlTriggerConstants.SysChangeVersionColumnName}) OR - l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND - (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}); + (l.{LeasesTableLeaseExpirationTimeColumnName} IS NULL AND + (l.{LeasesTableChangeVersionColumnName} IS NULL OR l.{LeasesTableChangeVersionColumnName} < c.{SysChangeVersionColumnName}) OR + l.{LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND + (l.{LeasesTableAttemptCountColumnName} IS NULL OR l.{LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}); "; return new SqlCommand(getUnprocessedChangesQuery, connection); @@ -813,8 +817,8 @@ private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransa IEnumerable cteColumnDefinitions = this._primaryKeyColumns .Select(c => $"{c.name.AsBracketQuotedString()} {c.type}") // These are the internal column values that we use. Note that we use SYS_CHANGE_VERSION because that's - // the new version - the _az_func_ChangeVersion has the old version - .Concat(new string[] { $"{SqlTriggerConstants.SysChangeVersionColumnName} bigint", $"{SqlTriggerConstants.LeasesTableAttemptCountColumnName} int" }); + // the new version - the _az_func_ChangeVersion has the old version + .Concat(new string[] { $"{SysChangeVersionColumnName} bigint", $"{LeasesTableAttemptCountColumnName} int" }); IEnumerable bracketedPrimaryKeys = this._primaryKeyColumns.Select(p => p.name.AsBracketQuotedString()); // Create the query that the merge statement will match the rows on @@ -832,11 +836,11 @@ AS NewData {primaryKeyMatchingQuery} WHEN MATCHED THEN UPDATE SET - {SqlTriggerConstants.LeasesTableChangeVersionColumnName} = NewData.{SqlTriggerConstants.SysChangeVersionColumnName}, - {SqlTriggerConstants.LeasesTableAttemptCountColumnName} = ExistingData.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} + 1, - {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + {LeasesTableChangeVersionColumnName} = NewData.{SysChangeVersionColumnName}, + {LeasesTableAttemptCountColumnName} = ExistingData.{LeasesTableAttemptCountColumnName} + 1, + {LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) WHEN NOT MATCHED THEN - INSERT VALUES ({string.Join(",", bracketedPrimaryKeys.Select(k => $"NewData.{k}"))}, NewData.{SqlTriggerConstants.SysChangeVersionColumnName}, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()));"; + INSERT VALUES ({string.Join(",", bracketedPrimaryKeys.Select(k => $"NewData.{k}"))}, NewData.{SysChangeVersionColumnName}, 1, DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()));"; var command = new SqlCommand(query, connection, transaction); SqlParameter par = command.Parameters.Add(rowDataParameter, SqlDbType.NVarChar, -1); @@ -856,7 +860,7 @@ private SqlCommand BuildRenewLeasesCommand(SqlConnection connection) string renewLeasesQuery = $@" UPDATE {this._leasesTableName} WITH (TABLOCKX) - SET {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) + SET {LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) WHERE {matchCondition}; "; @@ -876,19 +880,19 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa for (int rowIndex = 0; rowIndex < this._rows.Count; rowIndex++) { - string changeVersion = this._rows[rowIndex][SqlTriggerConstants.SysChangeVersionColumnName].ToString(); + string changeVersion = this._rows[rowIndex][SysChangeVersionColumnName].ToString(); releaseLeasesQuery.Append($@" - SELECT @current_change_version = {SqlTriggerConstants.LeasesTableChangeVersionColumnName} + SELECT @current_change_version = {LeasesTableChangeVersionColumnName} FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {this._rowMatchConditions[rowIndex]}; IF @current_change_version <= {changeVersion} UPDATE {this._leasesTableName} WITH (TABLOCKX) SET - {SqlTriggerConstants.LeasesTableChangeVersionColumnName} = {changeVersion}, - {SqlTriggerConstants.LeasesTableAttemptCountColumnName} = 0, - {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} = NULL + {LeasesTableChangeVersionColumnName} = {changeVersion}, + {LeasesTableAttemptCountColumnName} = 0, + {LeasesTableLeaseExpirationTimeColumnName} = NULL WHERE {this._rowMatchConditions[rowIndex]}; "); } @@ -912,28 +916,28 @@ private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, Sql string updateTablesPostInvocationQuery = $@" DECLARE @current_last_sync_version bigint; SELECT @current_last_sync_version = LastSyncVersion - FROM {SqlTriggerConstants.GlobalStateTableName} + FROM {GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; DECLARE @unprocessed_changes bigint; SELECT @unprocessed_changes = COUNT(*) FROM ( - SELECT c.{SqlTriggerConstants.SysChangeVersionColumnName} + SELECT c.{SysChangeVersionColumnName} FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_sync_version) AS c LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} WHERE - c.{SqlTriggerConstants.SysChangeVersionColumnName} <= {newLastSyncVersion} AND - ((l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} IS NULL OR - l.{SqlTriggerConstants.LeasesTableChangeVersionColumnName} != c.{SqlTriggerConstants.SysChangeVersionColumnName} OR - l.{SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} IS NOT NULL) AND - (l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} IS NULL OR l.{SqlTriggerConstants.LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}))) AS Changes + c.{SysChangeVersionColumnName} <= {newLastSyncVersion} AND + ((l.{LeasesTableChangeVersionColumnName} IS NULL OR + l.{LeasesTableChangeVersionColumnName} != c.{SysChangeVersionColumnName} OR + l.{LeasesTableLeaseExpirationTimeColumnName} IS NOT NULL) AND + (l.{LeasesTableAttemptCountColumnName} IS NULL OR l.{LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount}))) AS Changes IF @unprocessed_changes = 0 AND @current_last_sync_version < {newLastSyncVersion} BEGIN - UPDATE {SqlTriggerConstants.GlobalStateTableName} + UPDATE {GlobalStateTableName} SET LastSyncVersion = {newLastSyncVersion} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; - DELETE FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {SqlTriggerConstants.LeasesTableChangeVersionColumnName} <= {newLastSyncVersion}; + DELETE FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {LeasesTableChangeVersionColumnName} <= {newLastSyncVersion}; END "; diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 187a00383..bab3ba25d 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry; using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry; +using static Microsoft.Azure.WebJobs.Extensions.Sql.SqlTriggerConstants; using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Listeners; using Microsoft.Azure.WebJobs.Host.Scale; @@ -77,11 +78,11 @@ public SqlTriggerListener(string connectionString, string tableName, string user // In case converting from string to int is not possible from the user input. try { - configuredMaxChangesPerWorker = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_MaxChangesPerWorker); + configuredMaxChangesPerWorker = configuration.GetValue(ConfigKey_SqlTrigger_MaxChangesPerWorker); } catch (Exception ex) { - this._logger.LogError($"Failed to resolve integer value from user configured setting '{SqlTriggerConstants.ConfigKey_SqlTrigger_MaxChangesPerWorker}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); + this._logger.LogError($"Failed to resolve integer value from user configured setting '{ConfigKey_SqlTrigger_MaxChangesPerWorker}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); TelemetryInstance.TrackException(TelemetryErrorName.InvalidConfigurationValue, ex, this._telemetryProps); configuredMaxChangesPerWorker = DefaultMaxChangesPerWorker; @@ -126,7 +127,7 @@ public async Task StartAsync(CancellationToken cancellationToken) IReadOnlyList<(string name, string type)> primaryKeyColumns = await this.GetPrimaryKeyColumnsAsync(connection, userTableId, cancellationToken); IReadOnlyList userTableColumns = await this.GetUserTableColumnsAsync(connection, userTableId, cancellationToken); - string leasesTableName = string.Format(CultureInfo.InvariantCulture, SqlTriggerConstants.LeasesTableNameFormat, $"{this._userFunctionId}_{userTableId}"); + string leasesTableName = string.Format(CultureInfo.InvariantCulture, LeasesTableNameFormat, $"{this._userFunctionId}_{userTableId}"); this._telemetryProps[TelemetryPropertyName.LeasesTableName] = leasesTableName; var transactionSw = Stopwatch.StartNew(); @@ -328,7 +329,7 @@ FROM sys.columns AS c throw new InvalidOperationException($"Found column(s) with unsupported type(s): {columnNamesAndTypes} in table: '{this._userTable.FullName}'."); } - var conflictingColumnNames = userTableColumns.Intersect(SqlTriggerConstants.ReservedColumnNames).ToList(); + var conflictingColumnNames = userTableColumns.Intersect(ReservedColumnNames).ToList(); if (conflictingColumnNames.Count > 0) { @@ -352,8 +353,8 @@ FROM sys.columns AS c private async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { string createSchemaQuery = $@" - IF SCHEMA_ID(N'{SqlTriggerConstants.SchemaName}') IS NULL - EXEC ('CREATE SCHEMA {SqlTriggerConstants.SchemaName}'); + IF SCHEMA_ID(N'{SchemaName}') IS NULL + EXEC ('CREATE SCHEMA {SchemaName}'); "; this._logger.LogDebugWithThreadId($"BEGIN CreateSchema Query={createSchemaQuery}"); @@ -377,8 +378,8 @@ IF SCHEMA_ID(N'{SqlTriggerConstants.SchemaName}') IS NULL private async Task CreateGlobalStateTableAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { string createGlobalStateTableQuery = $@" - IF OBJECT_ID(N'{SqlTriggerConstants.GlobalStateTableName}', 'U') IS NULL - CREATE TABLE {SqlTriggerConstants.GlobalStateTableName} ( + IF OBJECT_ID(N'{GlobalStateTableName}', 'U') IS NULL + CREATE TABLE {GlobalStateTableName} ( UserFunctionID char(16) NOT NULL, UserTableID int NOT NULL, LastSyncVersion bigint NOT NULL, @@ -431,10 +432,10 @@ private async Task InsertGlobalStateTableRowAsync(SqlConnection connection string insertRowGlobalStateTableQuery = $@" IF NOT EXISTS ( - SELECT * FROM {SqlTriggerConstants.GlobalStateTableName} + SELECT * FROM {GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {userTableId} ) - INSERT INTO {SqlTriggerConstants.GlobalStateTableName} + INSERT INTO {GlobalStateTableName} VALUES ('{this._userFunctionId}', {userTableId}, {(long)minValidVersion}); "; @@ -473,9 +474,9 @@ private async Task CreateLeasesTableAsync( IF OBJECT_ID(N'{leasesTableName}', 'U') IS NULL CREATE TABLE {leasesTableName} ( {primaryKeysWithTypes}, - {SqlTriggerConstants.LeasesTableChangeVersionColumnName} bigint NOT NULL, - {SqlTriggerConstants.LeasesTableAttemptCountColumnName} int NOT NULL, - {SqlTriggerConstants.LeasesTableLeaseExpirationTimeColumnName} datetime2, + {LeasesTableChangeVersionColumnName} bigint NOT NULL, + {LeasesTableAttemptCountColumnName} int NOT NULL, + {LeasesTableLeaseExpirationTimeColumnName} datetime2, PRIMARY KEY ({primaryKeys}) ); "; From 43480cd3306d8ea59514e19fac6e9f15cb1222a1 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 4 Nov 2022 10:03:51 -0700 Subject: [PATCH 64/77] initial --- performance/SqlBindingBenchmarks.cs | 10 ++- ...iggerBindingPerformance_Parallelization.cs | 62 +++++++++++++++++++ .../SqlTriggerPerformance_BatchOverride.cs | 38 ++++++++++++ .../SqlTriggerPerformance_Overrides.cs | 51 +++++++++++++++ ...ggerPerformance_PollingIntervalOverride.cs | 38 ++++++++++++ 5 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 performance/SqlTriggerBindingPerformance_Parallelization.cs create mode 100644 performance/SqlTriggerPerformance_BatchOverride.cs create mode 100644 performance/SqlTriggerPerformance_Overrides.cs create mode 100644 performance/SqlTriggerPerformance_PollingIntervalOverride.cs diff --git a/performance/SqlBindingBenchmarks.cs b/performance/SqlBindingBenchmarks.cs index 7917e17e2..8f2acc887 100644 --- a/performance/SqlBindingBenchmarks.cs +++ b/performance/SqlBindingBenchmarks.cs @@ -9,9 +9,13 @@ public class SqlBindingPerformance { public static void Main() { - BenchmarkRunner.Run(); - BenchmarkRunner.Run(); - BenchmarkRunner.Run(); + // BenchmarkRunner.Run(); + // BenchmarkRunner.Run(); + // BenchmarkRunner.Run(); + // BenchmarkRunner.Run(); + // BenchmarkRunner.Run(); + // BenchmarkRunner.Run(); + BenchmarkRunner.Run(); } } } \ No newline at end of file diff --git a/performance/SqlTriggerBindingPerformance_Parallelization.cs b/performance/SqlTriggerBindingPerformance_Parallelization.cs new file mode 100644 index 000000000..3ccee8e7a --- /dev/null +++ b/performance/SqlTriggerBindingPerformance_Parallelization.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance +{ + [MemoryDiagnoser] + public class SqlTriggerBindingPerformance_Parallelization : SqlTriggerBindingPerformanceTestBase + { + [Benchmark] + [Arguments(2)] + [Arguments(5)] + [Arguments(10)] + public async Task MultiHost(int hostCount) + { + for (int i = 0; i < hostCount; ++i) + { + this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); + } + + int firstId = 1; + int lastId = 90; + await this.WaitForProductChanges( + firstId, + lastId, + SqlChangeOperation.Insert, + () => { this.InsertProducts(firstId, lastId); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + this.GetBatchProcessingTimeout(firstId, lastId)); + + firstId = 1; + lastId = 60; + // All table columns (not just the columns that were updated) would be returned for update operation. + await this.WaitForProductChanges( + firstId, + lastId, + SqlChangeOperation.Update, + () => { this.UpdateProducts(firstId, lastId); return Task.CompletedTask; }, + id => $"Updated Product {id}", + id => id * 100, + this.GetBatchProcessingTimeout(firstId, lastId)); + + firstId = 31; + lastId = 90; + // The properties corresponding to non-primary key columns would be set to the C# type's default values + // (null and 0) for delete operation. + await this.WaitForProductChanges( + firstId, + lastId, + SqlChangeOperation.Delete, + () => { this.DeleteProducts(firstId, lastId); return Task.CompletedTask; }, + _ => null, + _ => 0, + this.GetBatchProcessingTimeout(firstId, lastId)); + } + } +} \ No newline at end of file diff --git a/performance/SqlTriggerPerformance_BatchOverride.cs b/performance/SqlTriggerPerformance_BatchOverride.cs new file mode 100644 index 000000000..ad232f215 --- /dev/null +++ b/performance/SqlTriggerPerformance_BatchOverride.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using BenchmarkDotNet.Attributes; +using System.Collections.Generic; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance +{ + [MemoryDiagnoser] + public class SqlTriggerBindingPerformance_BatchOverride : SqlTriggerBindingPerformanceTestBase + { + [Benchmark] + [Arguments(10, 1000)] + [Arguments(100, 1000)] + [Arguments(1000, 1000)] + [Arguments(5000, 1000)] + public async Task Run(int count, int batchSize) + { + this.StartFunctionHost( + nameof(ProductsTrigger), + SupportedLanguages.CSharp, + environmentVariables: new Dictionary() { + { "Sql_Trigger_BatchSize", batchSize.ToString() } + }); + await this.WaitForProductChanges( + 1, + count, + SqlChangeOperation.Insert, + () => { this.InsertProducts(1, count); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + this.GetBatchProcessingTimeout(1, count, batchSize: batchSize)); + } + } +} \ No newline at end of file diff --git a/performance/SqlTriggerPerformance_Overrides.cs b/performance/SqlTriggerPerformance_Overrides.cs new file mode 100644 index 000000000..41e857018 --- /dev/null +++ b/performance/SqlTriggerPerformance_Overrides.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using BenchmarkDotNet.Attributes; +using System.Collections.Generic; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance +{ + [MemoryDiagnoser] + public class SqlTriggerPerformance_Overrides : SqlTriggerBindingPerformanceTestBase + { + [Benchmark] + [Arguments(10, 1000, 500)] + [Arguments(10, 1000, 100)] + [Arguments(10, 1000, 10)] + [Arguments(10, 1000, 1)] + [Arguments(100, 1000, 500)] + [Arguments(100, 1000, 100)] + [Arguments(100, 1000, 10)] + [Arguments(100, 1000, 1)] + [Arguments(1000, 1000, 500)] + [Arguments(1000, 1000, 100)] + [Arguments(1000, 1000, 10)] + [Arguments(1000, 1000, 1)] + [Arguments(5000, 1000, 500)] + [Arguments(5000, 1000, 100)] + [Arguments(5000, 1000, 10)] + [Arguments(5000, 1000, 1)] + public async Task Run(int count, int batchSize, int pollingIntervalMs) + { + this.StartFunctionHost( + nameof(ProductsTrigger), + SupportedLanguages.CSharp, + environmentVariables: new Dictionary() { + { "Sql_Trigger_BatchSize", batchSize.ToString() }, + { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } + }); + await this.WaitForProductChanges( + 1, + count, + SqlChangeOperation.Insert, + () => { this.InsertProducts(1, count); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + this.GetBatchProcessingTimeout(1, count, batchSize: batchSize)); + } + } +} \ No newline at end of file diff --git a/performance/SqlTriggerPerformance_PollingIntervalOverride.cs b/performance/SqlTriggerPerformance_PollingIntervalOverride.cs new file mode 100644 index 000000000..b53d5d19d --- /dev/null +++ b/performance/SqlTriggerPerformance_PollingIntervalOverride.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.TriggerBindingSamples; +using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common; +using BenchmarkDotNet.Attributes; +using System.Collections.Generic; + +namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance +{ + [MemoryDiagnoser] + public class SqlTriggerBindingPerformance_PollingIntervalOverride : SqlTriggerBindingPerformanceTestBase + { + [Benchmark] + [Arguments(1000, 500)] + [Arguments(1000, 100)] + [Arguments(1000, 10)] + [Arguments(1000, 1)] + public async Task Run(int count, int pollingIntervalMs) + { + this.StartFunctionHost( + nameof(ProductsTrigger), + SupportedLanguages.CSharp, + environmentVariables: new Dictionary() { + { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } + }); + await this.WaitForProductChanges( + 1, + count, + SqlChangeOperation.Insert, + () => { this.InsertProducts(1, count); return Task.CompletedTask; }, + id => $"Product {id}", + id => id * 100, + this.GetBatchProcessingTimeout(1, count, pollingIntervalMs: pollingIntervalMs)); + } + } +} \ No newline at end of file From 7367c92dc448653fb43ca00b2b7c364a65163abc Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 4 Nov 2022 12:48:59 -0700 Subject: [PATCH 65/77] updates --- performance/SqlBindingBenchmarks.cs | 12 ++++++------ .../SqlTriggerBindingPerformance_Parallelization.cs | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/performance/SqlBindingBenchmarks.cs b/performance/SqlBindingBenchmarks.cs index 8f2acc887..9cceb2797 100644 --- a/performance/SqlBindingBenchmarks.cs +++ b/performance/SqlBindingBenchmarks.cs @@ -9,12 +9,12 @@ public class SqlBindingPerformance { public static void Main() { - // BenchmarkRunner.Run(); - // BenchmarkRunner.Run(); - // BenchmarkRunner.Run(); - // BenchmarkRunner.Run(); - // BenchmarkRunner.Run(); - // BenchmarkRunner.Run(); + BenchmarkRunner.Run(); + BenchmarkRunner.Run(); + BenchmarkRunner.Run(); + BenchmarkRunner.Run(); + BenchmarkRunner.Run(); + BenchmarkRunner.Run(); BenchmarkRunner.Run(); } } diff --git a/performance/SqlTriggerBindingPerformance_Parallelization.cs b/performance/SqlTriggerBindingPerformance_Parallelization.cs index 3ccee8e7a..20f9d6e66 100644 --- a/performance/SqlTriggerBindingPerformance_Parallelization.cs +++ b/performance/SqlTriggerBindingPerformance_Parallelization.cs @@ -14,7 +14,6 @@ public class SqlTriggerBindingPerformance_Parallelization : SqlTriggerBindingPer [Benchmark] [Arguments(2)] [Arguments(5)] - [Arguments(10)] public async Task MultiHost(int hostCount) { for (int i = 0; i < hostCount; ++i) From 7d4fd75d5699dc677f7a72c303da98a3f8ebb0ad Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 4 Nov 2022 13:31:26 -0700 Subject: [PATCH 66/77] Use app locks for all transactions --- src/TriggerBinding/SqlTableChangeMonitor.cs | 34 +++++++++++++------ src/TriggerBinding/SqlTriggerConstants.cs | 36 +++++++++++++++++++++ 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index eb67ef52f..e9f4e6027 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -718,6 +718,8 @@ private static SqlChangeOperation GetChangeOperation(IReadOnlyDictionary $"c.{col.name.AsBracketQuotedString()} = l.{col.name.AsBracketQuotedString()}")); string getChangesQuery = $@" + {AppLockStatements} + DECLARE @last_sync_version bigint; SELECT @last_sync_version = LastSyncVersion FROM {GlobalStateTableName} @@ -761,7 +765,7 @@ SELECT TOP {this._batchSize} l.{LeasesTableAttemptCountColumnName}, l.{LeasesTableLeaseExpirationTimeColumnName} FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c - LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} + LEFT OUTER JOIN {this._leasesTableName} AS l ON {leasesTableJoinCondition} LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCondition} WHERE (l.{LeasesTableLeaseExpirationTimeColumnName} IS NULL AND @@ -785,6 +789,8 @@ private SqlCommand BuildGetUnprocessedChangesCommand(SqlConnection connection) string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.name.AsBracketQuotedString()} = l.{col.name.AsBracketQuotedString()}")); string getUnprocessedChangesQuery = $@" + {AppLockStatements} + DECLARE @last_sync_version bigint; SELECT @last_sync_version = LastSyncVersion FROM {GlobalStateTableName} @@ -792,7 +798,7 @@ private SqlCommand BuildGetUnprocessedChangesCommand(SqlConnection connection) SELECT COUNT_BIG(*) FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c - LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} + LEFT OUTER JOIN {this._leasesTableName} AS l ON {leasesTableJoinCondition} WHERE (l.{LeasesTableLeaseExpirationTimeColumnName} IS NULL AND (l.{LeasesTableChangeVersionColumnName} IS NULL OR l.{LeasesTableChangeVersionColumnName} < c.{SysChangeVersionColumnName}) OR @@ -827,8 +833,10 @@ private SqlCommand BuildAcquireLeasesCommand(SqlConnection connection, SqlTransa const string rowDataParameter = "@rowData"; // Create the merge query that will either update the rows that already exist or insert a new one if it doesn't exist string query = $@" + {AppLockStatements} + WITH {acquireLeasesCte} AS ( SELECT * FROM OPENJSON(@rowData) WITH ({string.Join(",", cteColumnDefinitions)}) ) - MERGE INTO {this._leasesTableName} WITH (TABLOCKX) + MERGE INTO {this._leasesTableName} AS ExistingData USING {acquireLeasesCte} AS NewData @@ -859,7 +867,9 @@ private SqlCommand BuildRenewLeasesCommand(SqlConnection connection) string matchCondition = string.Join(" OR ", this._rowMatchConditions.Take(this._rows.Count)); string renewLeasesQuery = $@" - UPDATE {this._leasesTableName} WITH (TABLOCKX) + {AppLockStatements} + + UPDATE {this._leasesTableName} SET {LeasesTableLeaseExpirationTimeColumnName} = DATEADD(second, {LeaseIntervalInSeconds}, SYSDATETIME()) WHERE {matchCondition}; "; @@ -876,7 +886,11 @@ private SqlCommand BuildRenewLeasesCommand(SqlConnection connection) /// The SqlCommand populated with the query and appropriate parameters private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransaction transaction) { - var releaseLeasesQuery = new StringBuilder("DECLARE @current_change_version bigint;\n"); + var releaseLeasesQuery = new StringBuilder( +$@"{AppLockStatements} + +DECLARE @current_change_version bigint; +"); for (int rowIndex = 0; rowIndex < this._rows.Count; rowIndex++) { @@ -884,11 +898,11 @@ private SqlCommand BuildReleaseLeasesCommand(SqlConnection connection, SqlTransa releaseLeasesQuery.Append($@" SELECT @current_change_version = {LeasesTableChangeVersionColumnName} - FROM {this._leasesTableName} WITH (TABLOCKX) + FROM {this._leasesTableName} WHERE {this._rowMatchConditions[rowIndex]}; IF @current_change_version <= {changeVersion} - UPDATE {this._leasesTableName} WITH (TABLOCKX) + UPDATE {this._leasesTableName} SET {LeasesTableChangeVersionColumnName} = {changeVersion}, {LeasesTableAttemptCountColumnName} = 0, @@ -914,6 +928,8 @@ private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, Sql string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.name.AsBracketQuotedString()} = l.{col.name.AsBracketQuotedString()}")); string updateTablesPostInvocationQuery = $@" + {AppLockStatements} + DECLARE @current_last_sync_version bigint; SELECT @current_last_sync_version = LastSyncVersion FROM {GlobalStateTableName} @@ -923,7 +939,7 @@ private SqlCommand BuildUpdateTablesPostInvocation(SqlConnection connection, Sql SELECT @unprocessed_changes = COUNT(*) FROM ( SELECT c.{SysChangeVersionColumnName} FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_sync_version) AS c - LEFT OUTER JOIN {this._leasesTableName} AS l WITH (TABLOCKX) ON {leasesTableJoinCondition} + LEFT OUTER JOIN {this._leasesTableName} AS l ON {leasesTableJoinCondition} WHERE c.{SysChangeVersionColumnName} <= {newLastSyncVersion} AND ((l.{LeasesTableChangeVersionColumnName} IS NULL OR @@ -937,7 +953,7 @@ FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @current_last_ SET LastSyncVersion = {newLastSyncVersion} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId}; - DELETE FROM {this._leasesTableName} WITH (TABLOCKX) WHERE {LeasesTableChangeVersionColumnName} <= {newLastSyncVersion}; + DELETE FROM {this._leasesTableName} WHERE {LeasesTableChangeVersionColumnName} <= {newLastSyncVersion}; END "; diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index 533438500..1c06c0a35 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -29,5 +29,41 @@ internal static class SqlTriggerConstants public const string ConfigKey_SqlTrigger_BatchSize = "Sql_Trigger_BatchSize"; public const string ConfigKey_SqlTrigger_PollingInterval = "Sql_Trigger_PollingIntervalMs"; public const string ConfigKey_SqlTrigger_MaxChangesPerWorker = "Sql_Trigger_MaxChangesPerWorker"; + + public const string AppLockResource = "_az_func_Trigger"; + /// + /// Timeout for acquiring the application lock - 30sec chosen as a reasonable value to ensure we aren't + /// hanging infinitely while also giving plenty of time for the blocking transaction to complete. + /// + public const int AppLockTimeoutMs = 30000; + + /// + /// T-SQL statements for getting an application lock. This is used to prevent deadlocks - primarily when multiple instances + /// of a function are running in parallel. + /// + /// The trigger heavily uses transactions to ensure atomic changes, that way if an error occurs during any step of a process we aren't left + /// with an incomplete state. Because of this locks are placed on rows that are read/modified during the transaction, but the lock isn't + /// applied until the statement itself is executed. Some transactions have many statements executed in a row that touch a number of different + /// tables so it's very easy for two transactions to get in a deadlock depending on the speed they execute their statements and the order they + /// are processed in. + /// + /// So to avoid this we use application locks to ensure that anytime we enter a transaction we first guarantee that we're the only transaction + /// currently making any changes to the tables, which means that we're guaranteed not to have any deadlocks - albeit at the cost of speed. This + /// is acceptable for now, although further investigation could be done into using multiple resources to lock on (such as a different one for each + /// table) to increase the parallelization of the transactions. + /// + /// See the following articles for more information on locking in MSSQL + /// https://learn.microsoft.com/sql/relational-databases/sql-server-transaction-locking-and-row-versioning-guide + /// https://learn.microsoft.com/sql/t-sql/statements/set-transaction-isolation-level-transact-sql + /// https://learn.microsoft.com/sql/relational-databases/system-stored-procedures/sp-getapplock-transact-sql + /// + public static readonly string AppLockStatements = $@"DECLARE @result int; + EXEC @result = sp_getapplock @Resource = '{AppLockResource}', + @LockMode = 'Exclusive', + @LockTimeout = {AppLockTimeoutMs} + IF @result < 0 + BEGIN + RAISERROR('Unable to acquire exclusive lock on {AppLockResource}. Result = %d', 16, 1, @result) + END;"; } } \ No newline at end of file From 14fa06144758e1a197265bbbcf9758807029c737 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 4 Nov 2022 13:55:20 -0700 Subject: [PATCH 67/77] Add resource comment --- src/TriggerBinding/SqlTriggerConstants.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index 1c06c0a35..c45cb97b3 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -30,6 +30,12 @@ internal static class SqlTriggerConstants public const string ConfigKey_SqlTrigger_PollingInterval = "Sql_Trigger_PollingIntervalMs"; public const string ConfigKey_SqlTrigger_MaxChangesPerWorker = "Sql_Trigger_MaxChangesPerWorker"; + /// + /// The resource name to use for getting the application lock. We use the same resource name for all instances + /// of the function because there is some shared state across all the functions. + /// + /// A future improvement could be to make unique application locks for each FuncId/TableId combination so that functions + /// working on different tables aren't blocking each other public const string AppLockResource = "_az_func_Trigger"; /// /// Timeout for acquiring the application lock - 30sec chosen as a reasonable value to ensure we aren't From 6469902b6c3434ad488d3655917a2a222617e6a2 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Sun, 6 Nov 2022 17:57:12 -0800 Subject: [PATCH 68/77] comma --- src/TriggerBinding/SqlTriggerConstants.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index c45cb97b3..7e1d62651 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -48,7 +48,7 @@ internal static class SqlTriggerConstants /// of a function are running in parallel. /// /// The trigger heavily uses transactions to ensure atomic changes, that way if an error occurs during any step of a process we aren't left - /// with an incomplete state. Because of this locks are placed on rows that are read/modified during the transaction, but the lock isn't + /// with an incomplete state. Because of this, locks are placed on rows that are read/modified during the transaction, but the lock isn't /// applied until the statement itself is executed. Some transactions have many statements executed in a row that touch a number of different /// tables so it's very easy for two transactions to get in a deadlock depending on the speed they execute their statements and the order they /// are processed in. @@ -72,4 +72,4 @@ IF @result < 0 RAISERROR('Unable to acquire exclusive lock on {AppLockResource}. Result = %d', 16, 1, @result) END;"; } -} \ No newline at end of file +} From 0078e6d299d408e56ec3780d1ec618d91d1e4e5f Mon Sep 17 00:00:00 2001 From: AmeyaRele <35621237+AmeyaRele@users.noreply.github.com> Date: Mon, 7 Nov 2022 22:55:02 +0530 Subject: [PATCH 69/77] Throw exception for scaling configuration invalid value (#450) * Throw exception for scaling configuration invalid value * Change Exception type --- src/TriggerBinding/SqlTriggerListener.cs | 17 +++++------------ .../TriggerBinding/SqlTriggerListenerTests.cs | 16 ++++------------ 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index bab3ba25d..0887f5047 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -70,24 +70,17 @@ public SqlTriggerListener(string connectionString, string tableName, string user this._executor = executor ?? throw new ArgumentNullException(nameof(executor)); this._logger = logger ?? throw new ArgumentNullException(nameof(logger)); this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - int configuredMaxChangesPerWorker; + int? configuredMaxChangesPerWorker; // Do not convert the scale-monitor ID to lower-case string since SQL table names can be case-sensitive // depending on the collation of the current database. this._scaleMonitorDescriptor = new ScaleMonitorDescriptor($"{userFunctionId}-SqlTrigger-{tableName}"); - // In case converting from string to int is not possible from the user input. - try + configuredMaxChangesPerWorker = configuration.GetValue(ConfigKey_SqlTrigger_MaxChangesPerWorker); + this._maxChangesPerWorker = configuredMaxChangesPerWorker ?? DefaultMaxChangesPerWorker; + if (this._maxChangesPerWorker <= 0) { - configuredMaxChangesPerWorker = configuration.GetValue(ConfigKey_SqlTrigger_MaxChangesPerWorker); - } - catch (Exception ex) - { - this._logger.LogError($"Failed to resolve integer value from user configured setting '{ConfigKey_SqlTrigger_MaxChangesPerWorker}' due to exception: {ex.GetType()}. Exception message: {ex.Message}"); - TelemetryInstance.TrackException(TelemetryErrorName.InvalidConfigurationValue, ex, this._telemetryProps); - - configuredMaxChangesPerWorker = DefaultMaxChangesPerWorker; + throw new InvalidOperationException($"Invalid value for configuration setting '{ConfigKey_SqlTrigger_MaxChangesPerWorker}'. Ensure that the value is a positive integer."); } - this._maxChangesPerWorker = configuredMaxChangesPerWorker > 0 ? configuredMaxChangesPerWorker : DefaultMaxChangesPerWorker; } public void Cancel() diff --git a/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs b/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs index 8995bc18d..6f50e099f 100644 --- a/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs +++ b/test/Unit/TriggerBinding/SqlTriggerListenerTests.cs @@ -232,20 +232,12 @@ public void ScaleMonitorGetScaleStatus_UserConfiguredMaxChangesPerWorker_Respect [InlineData("-1")] [InlineData("0")] [InlineData("10000000000")] - public void ScaleMonitorGetScaleStatus_InvalidUserConfiguredMaxChangesPerWorker_UsesDefaultValue(string maxChangesPerWorker) + public void InvalidUserConfiguredMaxChangesPerWorker(string maxChangesPerWorker) { - (IScaleMonitor monitor, _) = GetScaleMonitor(maxChangesPerWorker); - - ScaleStatusContext context; - ScaleStatus scaleStatus; - - context = GetScaleStatusContext(new int[] { 0, 0, 0, 0, 10000 }, 10); - scaleStatus = monitor.GetScaleStatus(context); - Assert.Equal(ScaleVote.None, scaleStatus.Vote); + (Mock mockLogger, List logMessages) = CreateMockLogger(); + Mock mockConfiguration = CreateMockConfiguration(maxChangesPerWorker); - context = GetScaleStatusContext(new int[] { 0, 0, 0, 0, 10001 }, 10); - scaleStatus = monitor.GetScaleStatus(context); - Assert.Equal(ScaleVote.ScaleOut, scaleStatus.Vote); + Assert.Throws(() => new SqlTriggerListener("testConnectionString", "testTableName", "testUserFunctionId", Mock.Of(), mockLogger.Object, mockConfiguration.Object)); } private static IScaleMonitor GetScaleMonitor(string tableName, string userFunctionId) From f48e331bb6f8acdb9952d75858263c0a30d27f8c Mon Sep 17 00:00:00 2001 From: AmeyaRele <35621237+AmeyaRele@users.noreply.github.com> Date: Tue, 8 Nov 2022 01:39:23 +0530 Subject: [PATCH 70/77] Catch SqlException for existing object error (#456) * Catch SqlException for existing object error * Update src/TriggerBinding/SqlTriggerListener.cs Co-authored-by: Charles Gagnon Co-authored-by: Charles Gagnon --- src/Telemetry/Telemetry.cs | 1 + src/TriggerBinding/SqlTriggerListener.cs | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index 91dd340c9..b4bd4c7bc 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -408,6 +408,7 @@ public enum TelemetryErrorName { ConsumeChangesLoop, Convert, + CreateSchema, FlushAsync, GetCaseSensitivity, GetChanges, diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 0887f5047..8c9961964 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -354,7 +354,29 @@ IF SCHEMA_ID(N'{SchemaName}') IS NULL using (var createSchemaCommand = new SqlCommand(createSchemaQuery, connection, transaction)) { var stopwatch = Stopwatch.StartNew(); - await createSchemaCommand.ExecuteNonQueryAsync(cancellationToken); + + try + { + await createSchemaCommand.ExecuteNonQueryAsync(cancellationToken); + } + catch (Exception ex) + { + TelemetryInstance.TrackException(TelemetryErrorName.CreateSchema, ex, this._telemetryProps); + var sqlEx = ex as SqlException; + if (sqlEx?.Number == 2714) + { + // Error 2714 is for an object of that name already existing in the database. This generally shouldn't happen + // since we check for its existence in the statement but occasionally a race condition can make it so + // that multiple instances will try and create the schema at once. In that case we can just ignore the + // error since all we care about is that the schema exists at all. + this._logger.LogWarning($"Failed to create schema '{SchemaName}'. Exception message: {ex.Message} This is informational only, function startup will continue as normal."); + } + else + { + throw; + } + } + long durationMs = stopwatch.ElapsedMilliseconds; this._logger.LogDebugWithThreadId($"END CreateSchema Duration={durationMs}ms"); return durationMs; From ae8fe392f6f5f7e0d8a77c2dca980c512adb86cd Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Mon, 7 Nov 2022 15:25:56 -0800 Subject: [PATCH 71/77] Fix perf tests --- performance/.editorconfig | 3 +- performance/SqlTriggerBindingPerformance.cs | 1 + .../SqlTriggerBindingPerformanceTestBase.cs | 37 +++++++-------- ...iggerBindingPerformance_Parallelization.cs | 16 +++++-- .../SqlTriggerPerformance_BatchOverride.cs | 27 +++++++---- .../SqlTriggerPerformance_Overrides.cs | 46 ++++++++++--------- ...ggerPerformance_PollingIntervalOverride.cs | 22 +++++---- ...rosoft.Azure.WebJobs.Extensions.Sql.csproj | 1 + 8 files changed, 89 insertions(+), 64 deletions(-) diff --git a/performance/.editorconfig b/performance/.editorconfig index 0082aa2a0..77a834e7a 100644 --- a/performance/.editorconfig +++ b/performance/.editorconfig @@ -6,4 +6,5 @@ dotnet_diagnostic.CA1309.severity = silent # Use ordinal StringComparison - this isn't important for tests and just adds clutter dotnet_diagnostic.CA1305.severity = silent # Specify IFormatProvider - this isn't important for tests and just adds clutter dotnet_diagnostic.CA1707.severity = silent # Identifiers should not contain underscores - this helps make test names more readable -dotnet_diagnostic.CA2201.severity = silent # Do not raise reserved exception types - tests can throw whatever they want \ No newline at end of file +dotnet_diagnostic.CA2201.severity = silent # Do not raise reserved exception types - tests can throw whatever they want +dotnet_diagnostic.CA1051.severity = silent # Do not declare visible instance fields - doesn't matter for tests \ No newline at end of file diff --git a/performance/SqlTriggerBindingPerformance.cs b/performance/SqlTriggerBindingPerformance.cs index 233339f43..2c7413897 100644 --- a/performance/SqlTriggerBindingPerformance.cs +++ b/performance/SqlTriggerBindingPerformance.cs @@ -14,6 +14,7 @@ public class SqlTriggerBindingPerformance : SqlTriggerBindingPerformanceTestBase [GlobalSetup] public void GlobalSetup() { + this.SetChangeTrackingForTable("Products", true); this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); } diff --git a/performance/SqlTriggerBindingPerformanceTestBase.cs b/performance/SqlTriggerBindingPerformanceTestBase.cs index d891118a0..79122043b 100644 --- a/performance/SqlTriggerBindingPerformanceTestBase.cs +++ b/performance/SqlTriggerBindingPerformanceTestBase.cs @@ -8,35 +8,30 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance { public class SqlTriggerBindingPerformanceTestBase : SqlTriggerBindingIntegrationTests { - [IterationSetup] - public void IterationSetup() - { - this.SetChangeTrackingForTable("Products", true); - } - [IterationCleanup] public void IterationCleanup() { - this.DisposeFunctionHosts(); + // Disable change tracking while cleaning up so we start off fresh for the next iteration this.SetChangeTrackingForTable("Products", false); // Delete all rows in Products table after each iteration so we start fresh each time this.ExecuteNonQuery("TRUNCATE TABLE Products"); - // Delete the leases table, otherwise we may end up getting blocked by leases from a previous run + // Clear the leases table, otherwise we may end up getting blocked by leases from a previous run this.ExecuteNonQuery(@"DECLARE @cmd varchar(100) -DECLARE cmds CURSOR FOR -SELECT 'DROP TABLE az_func.' + Name + '' -FROM sys.tables -WHERE Name LIKE 'Leases_%' + DECLARE cmds CURSOR FOR + SELECT 'TRUNCATE TABLE az_func.' + Name + '' + FROM sys.tables + WHERE Name LIKE 'Leases_%' -OPEN cmds -WHILE 1 = 1 -BEGIN - FETCH cmds INTO @cmd - IF @@fetch_status != 0 BREAK - EXEC(@cmd) -END -CLOSE cmds; -DEALLOCATE cmds"); + OPEN cmds + WHILE 1 = 1 + BEGIN + FETCH cmds INTO @cmd + IF @@fetch_status != 0 BREAK + EXEC(@cmd) + END + CLOSE cmds; + DEALLOCATE cmds"); + this.SetChangeTrackingForTable("Products", true); } [GlobalCleanup] diff --git a/performance/SqlTriggerBindingPerformance_Parallelization.cs b/performance/SqlTriggerBindingPerformance_Parallelization.cs index 20f9d6e66..4b46e8f60 100644 --- a/performance/SqlTriggerBindingPerformance_Parallelization.cs +++ b/performance/SqlTriggerBindingPerformance_Parallelization.cs @@ -11,16 +11,22 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance [MemoryDiagnoser] public class SqlTriggerBindingPerformance_Parallelization : SqlTriggerBindingPerformanceTestBase { - [Benchmark] - [Arguments(2)] - [Arguments(5)] - public async Task MultiHost(int hostCount) + [Params(2, 5)] + public int HostCount; + + [GlobalSetup] + public void GlobalSetup() { - for (int i = 0; i < hostCount; ++i) + this.SetChangeTrackingForTable("Products", true); + for (int i = 0; i < this.HostCount; ++i) { this.StartFunctionHost(nameof(ProductsTrigger), SupportedLanguages.CSharp); } + } + [Benchmark] + public async Task MultiHost() + { int firstId = 1; int lastId = 90; await this.WaitForProductChanges( diff --git a/performance/SqlTriggerPerformance_BatchOverride.cs b/performance/SqlTriggerPerformance_BatchOverride.cs index ad232f215..803774d29 100644 --- a/performance/SqlTriggerPerformance_BatchOverride.cs +++ b/performance/SqlTriggerPerformance_BatchOverride.cs @@ -12,19 +12,30 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance [MemoryDiagnoser] public class SqlTriggerBindingPerformance_BatchOverride : SqlTriggerBindingPerformanceTestBase { - [Benchmark] - [Arguments(10, 1000)] - [Arguments(100, 1000)] - [Arguments(1000, 1000)] - [Arguments(5000, 1000)] - public async Task Run(int count, int batchSize) + + [Params(100, 1000)] + public int BatchSize; + + [GlobalSetup] + public void GlobalSetup() { + this.SetChangeTrackingForTable("Products", true); this.StartFunctionHost( nameof(ProductsTrigger), SupportedLanguages.CSharp, environmentVariables: new Dictionary() { - { "Sql_Trigger_BatchSize", batchSize.ToString() } + { "Sql_Trigger_BatchSize", this.BatchSize.ToString() } }); + } + + [Benchmark] + [Arguments(0.1)] + [Arguments(0.5)] + [Arguments(1)] + [Arguments(5)] + public async Task Run(double numBatches) + { + int count = (int)(numBatches * this.BatchSize); await this.WaitForProductChanges( 1, count, @@ -32,7 +43,7 @@ await this.WaitForProductChanges( () => { this.InsertProducts(1, count); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - this.GetBatchProcessingTimeout(1, count, batchSize: batchSize)); + this.GetBatchProcessingTimeout(1, count, batchSize: this.BatchSize)); } } } \ No newline at end of file diff --git a/performance/SqlTriggerPerformance_Overrides.cs b/performance/SqlTriggerPerformance_Overrides.cs index 41e857018..b295ea6a4 100644 --- a/performance/SqlTriggerPerformance_Overrides.cs +++ b/performance/SqlTriggerPerformance_Overrides.cs @@ -12,32 +12,36 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance [MemoryDiagnoser] public class SqlTriggerPerformance_Overrides : SqlTriggerBindingPerformanceTestBase { - [Benchmark] - [Arguments(10, 1000, 500)] - [Arguments(10, 1000, 100)] - [Arguments(10, 1000, 10)] - [Arguments(10, 1000, 1)] - [Arguments(100, 1000, 500)] - [Arguments(100, 1000, 100)] - [Arguments(100, 1000, 10)] - [Arguments(100, 1000, 1)] - [Arguments(1000, 1000, 500)] - [Arguments(1000, 1000, 100)] - [Arguments(1000, 1000, 10)] - [Arguments(1000, 1000, 1)] - [Arguments(5000, 1000, 500)] - [Arguments(5000, 1000, 100)] - [Arguments(5000, 1000, 10)] - [Arguments(5000, 1000, 1)] - public async Task Run(int count, int batchSize, int pollingIntervalMs) + // [Params(1, 10, 100, 500)] + [Params(1, 10)] + public int PollingIntervalMs; + + [Params(500, 1000)] + // [Params(500, 1000, 2000)] + public int BatchSize; + + [GlobalSetup] + public void GlobalSetup() { + this.SetChangeTrackingForTable("Products", true); this.StartFunctionHost( nameof(ProductsTrigger), SupportedLanguages.CSharp, environmentVariables: new Dictionary() { - { "Sql_Trigger_BatchSize", batchSize.ToString() }, - { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } + { "Sql_Trigger_BatchSize", this.BatchSize.ToString() }, + { "Sql_Trigger_PollingIntervalMs", this.PollingIntervalMs.ToString() } }); + } + + [Benchmark] + [Arguments(0.1)] + [Arguments(0.5)] + //[Arguments(1)] + //[Arguments(5)] + // [Arguments(10)] + public async Task Run(double numBatches) + { + int count = (int)(numBatches * this.BatchSize); await this.WaitForProductChanges( 1, count, @@ -45,7 +49,7 @@ await this.WaitForProductChanges( () => { this.InsertProducts(1, count); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - this.GetBatchProcessingTimeout(1, count, batchSize: batchSize)); + this.GetBatchProcessingTimeout(1, count, batchSize: this.BatchSize)); } } } \ No newline at end of file diff --git a/performance/SqlTriggerPerformance_PollingIntervalOverride.cs b/performance/SqlTriggerPerformance_PollingIntervalOverride.cs index b53d5d19d..2d0bc0b68 100644 --- a/performance/SqlTriggerPerformance_PollingIntervalOverride.cs +++ b/performance/SqlTriggerPerformance_PollingIntervalOverride.cs @@ -12,19 +12,25 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance [MemoryDiagnoser] public class SqlTriggerBindingPerformance_PollingIntervalOverride : SqlTriggerBindingPerformanceTestBase { - [Benchmark] - [Arguments(1000, 500)] - [Arguments(1000, 100)] - [Arguments(1000, 10)] - [Arguments(1000, 1)] - public async Task Run(int count, int pollingIntervalMs) + [Params(1, 10, 100, 500, 2000)] + public int PollingIntervalMs; + + [GlobalSetup] + public void GlobalSetup() { + this.SetChangeTrackingForTable("Products", true); this.StartFunctionHost( nameof(ProductsTrigger), SupportedLanguages.CSharp, environmentVariables: new Dictionary() { - { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } + { "Sql_Trigger_PollingIntervalMs", this.PollingIntervalMs.ToString() } }); + } + + [Benchmark] + public async Task Run() + { + int count = SqlTableChangeMonitor.DefaultBatchSize * 2; await this.WaitForProductChanges( 1, count, @@ -32,7 +38,7 @@ await this.WaitForProductChanges( () => { this.InsertProducts(1, count); return Task.CompletedTask; }, id => $"Product {id}", id => id * 100, - this.GetBatchProcessingTimeout(1, count, pollingIntervalMs: pollingIntervalMs)); + this.GetBatchProcessingTimeout(1, count, pollingIntervalMs: this.PollingIntervalMs)); } } } \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Extensions.Sql.csproj b/src/Microsoft.Azure.WebJobs.Extensions.Sql.csproj index e7dc1d6b8..6b643b84a 100644 --- a/src/Microsoft.Azure.WebJobs.Extensions.Sql.csproj +++ b/src/Microsoft.Azure.WebJobs.Extensions.Sql.csproj @@ -37,6 +37,7 @@ + From 3e41737fcd953ebde8c1757f98b562c13c1f1727 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Mon, 7 Nov 2022 15:31:30 -0800 Subject: [PATCH 72/77] more --- performance/SqlTriggerBindingPerformanceTestBase.cs | 3 --- performance/SqlTriggerPerformance_Overrides.cs | 12 +++++------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/performance/SqlTriggerBindingPerformanceTestBase.cs b/performance/SqlTriggerBindingPerformanceTestBase.cs index 79122043b..ee6662c1e 100644 --- a/performance/SqlTriggerBindingPerformanceTestBase.cs +++ b/performance/SqlTriggerBindingPerformanceTestBase.cs @@ -11,8 +11,6 @@ public class SqlTriggerBindingPerformanceTestBase : SqlTriggerBindingIntegration [IterationCleanup] public void IterationCleanup() { - // Disable change tracking while cleaning up so we start off fresh for the next iteration - this.SetChangeTrackingForTable("Products", false); // Delete all rows in Products table after each iteration so we start fresh each time this.ExecuteNonQuery("TRUNCATE TABLE Products"); // Clear the leases table, otherwise we may end up getting blocked by leases from a previous run @@ -31,7 +29,6 @@ FETCH cmds INTO @cmd END CLOSE cmds; DEALLOCATE cmds"); - this.SetChangeTrackingForTable("Products", true); } [GlobalCleanup] diff --git a/performance/SqlTriggerPerformance_Overrides.cs b/performance/SqlTriggerPerformance_Overrides.cs index b295ea6a4..6b7faef62 100644 --- a/performance/SqlTriggerPerformance_Overrides.cs +++ b/performance/SqlTriggerPerformance_Overrides.cs @@ -12,12 +12,10 @@ namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance [MemoryDiagnoser] public class SqlTriggerPerformance_Overrides : SqlTriggerBindingPerformanceTestBase { - // [Params(1, 10, 100, 500)] - [Params(1, 10)] + [Params(1, 10, 100, 500)] public int PollingIntervalMs; - [Params(500, 1000)] - // [Params(500, 1000, 2000)] + [Params(500, 1000, 2000)] public int BatchSize; [GlobalSetup] @@ -36,9 +34,9 @@ public void GlobalSetup() [Benchmark] [Arguments(0.1)] [Arguments(0.5)] - //[Arguments(1)] - //[Arguments(5)] - // [Arguments(10)] + [Arguments(1)] + [Arguments(5)] + [Arguments(10)] public async Task Run(double numBatches) { int count = (int)(numBatches * this.BatchSize); From bcd00ddd24b1215075634fdd8bd6eb54cf81b864 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 8 Nov 2022 09:03:40 -0800 Subject: [PATCH 73/77] Add more app lock statements (#463) --- src/Telemetry/Telemetry.cs | 2 + src/TriggerBinding/SqlTriggerListener.cs | 52 +++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index b4bd4c7bc..79997b85a 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -408,6 +408,8 @@ public enum TelemetryErrorName { ConsumeChangesLoop, Convert, + CreateGlobalStateTable, + CreateLeasesTable, CreateSchema, FlushAsync, GetCaseSensitivity, diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index 8c9961964..f542958cd 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -346,6 +346,8 @@ FROM sys.columns AS c private async Task CreateSchemaAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { string createSchemaQuery = $@" + {AppLockStatements} + IF SCHEMA_ID(N'{SchemaName}') IS NULL EXEC ('CREATE SCHEMA {SchemaName}'); "; @@ -393,6 +395,8 @@ IF SCHEMA_ID(N'{SchemaName}') IS NULL private async Task CreateGlobalStateTableAsync(SqlConnection connection, SqlTransaction transaction, CancellationToken cancellationToken) { string createGlobalStateTableQuery = $@" + {AppLockStatements} + IF OBJECT_ID(N'{GlobalStateTableName}', 'U') IS NULL CREATE TABLE {GlobalStateTableName} ( UserFunctionID char(16) NOT NULL, @@ -406,7 +410,27 @@ PRIMARY KEY (UserFunctionID, UserTableID) using (var createGlobalStateTableCommand = new SqlCommand(createGlobalStateTableQuery, connection, transaction)) { var stopwatch = Stopwatch.StartNew(); - await createGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); + try + { + await createGlobalStateTableCommand.ExecuteNonQueryAsync(cancellationToken); + } + catch (Exception ex) + { + TelemetryInstance.TrackException(TelemetryErrorName.CreateGlobalStateTable, ex, this._telemetryProps); + var sqlEx = ex as SqlException; + if (sqlEx?.Number == 2714) + { + // Error 2714 is for an object of that name already existing in the database. This generally shouldn't happen + // since we check for its existence in the statement but occasionally a race condition can make it so + // that multiple instances will try and create the table at once. In that case we can just ignore the + // error since all we care about is that the table exists at all. + this._logger.LogWarning($"Failed to create global state table '{GlobalStateTableName}'. Exception message: {ex.Message} This is informational only, function startup will continue as normal."); + } + else + { + throw; + } + } long durationMs = stopwatch.ElapsedMilliseconds; this._logger.LogDebugWithThreadId($"END CreateGlobalStateTable Duration={durationMs}ms"); return durationMs; @@ -446,6 +470,8 @@ private async Task InsertGlobalStateTableRowAsync(SqlConnection connection this._logger.LogDebugWithThreadId($"END GetMinValidVersion MinValidVersion={minValidVersion}"); string insertRowGlobalStateTableQuery = $@" + {AppLockStatements} + IF NOT EXISTS ( SELECT * FROM {GlobalStateTableName} WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {userTableId} @@ -486,6 +512,8 @@ private async Task CreateLeasesTableAsync( string primaryKeys = string.Join(", ", primaryKeyColumns.Select(col => col.name.AsBracketQuotedString())); string createLeasesTableQuery = $@" + {AppLockStatements} + IF OBJECT_ID(N'{leasesTableName}', 'U') IS NULL CREATE TABLE {leasesTableName} ( {primaryKeysWithTypes}, @@ -500,7 +528,27 @@ PRIMARY KEY ({primaryKeys}) using (var createLeasesTableCommand = new SqlCommand(createLeasesTableQuery, connection, transaction)) { var stopwatch = Stopwatch.StartNew(); - await createLeasesTableCommand.ExecuteNonQueryAsync(cancellationToken); + try + { + await createLeasesTableCommand.ExecuteNonQueryAsync(cancellationToken); + } + catch (Exception ex) + { + TelemetryInstance.TrackException(TelemetryErrorName.CreateLeasesTable, ex, this._telemetryProps); + var sqlEx = ex as SqlException; + if (sqlEx?.Number == 2714) + { + // Error 2714 is for an object of that name already existing in the database. This generally shouldn't happen + // since we check for its existence in the statement but occasionally a race condition can make it so + // that multiple instances will try and create the table at once. In that case we can just ignore the + // error since all we care about is that the table exists at all. + this._logger.LogWarning($"Failed to create global state table '{leasesTableName}'. Exception message: {ex.Message} This is informational only, function startup will continue as normal."); + } + else + { + throw; + } + } long durationMs = stopwatch.ElapsedMilliseconds; this._logger.LogDebugWithThreadId($"END CreateLeasesTable Duration={durationMs}ms"); return durationMs; From 745875559baa15502e862c70ec6f756f21200ad9 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 8 Nov 2022 11:08:25 -0800 Subject: [PATCH 74/77] Validate monitor config values and send max changes configuration on startup (#467) * Validate monitor config values and send max changes configuration on startup * Add enum values --- src/Telemetry/Telemetry.cs | 2 ++ src/TriggerBinding/SqlTableChangeMonitor.cs | 19 +++++++++++++------ src/TriggerBinding/SqlTriggerListener.cs | 13 +++++++++++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/Telemetry/Telemetry.cs b/src/Telemetry/Telemetry.cs index 79997b85a..961c1b8ec 100644 --- a/src/Telemetry/Telemetry.cs +++ b/src/Telemetry/Telemetry.cs @@ -362,6 +362,7 @@ public enum TelemetryPropertyName ExceptionType, HasIdentityColumn, HasConfiguredBatchSize, + HasConfiguredMaxChangesPerWorker, HasConfiguredPollingInterval, LeasesTableName, QueryType, @@ -392,6 +393,7 @@ public enum TelemetryMeasureName GetPrimaryKeysDurationMs, GetUnprocessedChangesDurationMs, InsertGlobalStateTableRowDurationMs, + MaxChangesPerWorker, PollingIntervalMs, ReleaseLeasesDurationMs, RetryAttemptNumber, diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index e9f4e6027..4bd2dab74 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -130,19 +130,26 @@ public SqlTableChangeMonitor( int? configuredBatchSize = configuration.GetValue(ConfigKey_SqlTrigger_BatchSize); int? configuredPollingInterval = configuration.GetValue(ConfigKey_SqlTrigger_PollingInterval); this._batchSize = configuredBatchSize ?? this._batchSize; + if (this._batchSize <= 0) + { + throw new InvalidOperationException($"Invalid value for configuration setting '{ConfigKey_SqlTrigger_BatchSize}'. Ensure that the value is a positive integer."); + } this._pollingIntervalInMs = configuredPollingInterval ?? this._pollingIntervalInMs; - var monitorStartProps = new Dictionary(telemetryProps) + if (this._pollingIntervalInMs <= 0) { - { TelemetryPropertyName.HasConfiguredBatchSize, (configuredBatchSize != null).ToString() }, - { TelemetryPropertyName.HasConfiguredPollingInterval, (configuredPollingInterval != null).ToString() }, - }; + throw new InvalidOperationException($"Invalid value for configuration setting '{ConfigKey_SqlTrigger_PollingInterval}'. Ensure that the value is a positive integer."); + } TelemetryInstance.TrackEvent( TelemetryEventName.TriggerMonitorStart, - monitorStartProps, + new Dictionary(telemetryProps) { + { TelemetryPropertyName.HasConfiguredBatchSize, (configuredBatchSize != null).ToString() }, + { TelemetryPropertyName.HasConfiguredPollingInterval, (configuredPollingInterval != null).ToString() }, + }, new Dictionary() { { TelemetryMeasureName.BatchSize, this._batchSize }, { TelemetryMeasureName.PollingIntervalMs, this._pollingIntervalInMs } - }); + } + ); // Prep search-conditions that will be used besides WHERE clause to match table rows. this._rowMatchConditions = Enumerable.Range(0, this._batchSize) diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index f542958cd..c8d54ea3e 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -47,6 +47,7 @@ internal sealed class SqlTriggerListener : IListener, IScaleMonitor _telemetryProps = new Dictionary(); private readonly int _maxChangesPerWorker; + private readonly bool _hasConfiguredMaxChangesPerWorker = false; private SqlTableChangeMonitor _changeMonitor; private int _listenerState = ListenerNotStarted; @@ -74,13 +75,13 @@ public SqlTriggerListener(string connectionString, string tableName, string user // Do not convert the scale-monitor ID to lower-case string since SQL table names can be case-sensitive // depending on the collation of the current database. this._scaleMonitorDescriptor = new ScaleMonitorDescriptor($"{userFunctionId}-SqlTrigger-{tableName}"); - configuredMaxChangesPerWorker = configuration.GetValue(ConfigKey_SqlTrigger_MaxChangesPerWorker); this._maxChangesPerWorker = configuredMaxChangesPerWorker ?? DefaultMaxChangesPerWorker; if (this._maxChangesPerWorker <= 0) { throw new InvalidOperationException($"Invalid value for configuration setting '{ConfigKey_SqlTrigger_MaxChangesPerWorker}'. Ensure that the value is a positive integer."); } + this._hasConfiguredMaxChangesPerWorker = configuredMaxChangesPerWorker != null; } public void Cancel() @@ -105,7 +106,15 @@ public async Task StartAsync(CancellationToken cancellationToken) } this.InitializeTelemetryProps(); - TelemetryInstance.TrackEvent(TelemetryEventName.StartListenerStart, this._telemetryProps); + TelemetryInstance.TrackEvent( + TelemetryEventName.StartListenerStart, + new Dictionary(this._telemetryProps) { + { TelemetryPropertyName.HasConfiguredMaxChangesPerWorker, this._hasConfiguredMaxChangesPerWorker.ToString() } + }, + new Dictionary() { + { TelemetryMeasureName.MaxChangesPerWorker, this._maxChangesPerWorker } + } + ); try { From 607bd4d74f06211dd37e17ee68ef197306656889 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 8 Nov 2022 11:08:36 -0800 Subject: [PATCH 75/77] Move magic numbers to constants (#465) --- src/TriggerBinding/SqlTriggerConstants.cs | 9 +++- src/TriggerBinding/SqlTriggerListener.cs | 57 +++++++++++++---------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/TriggerBinding/SqlTriggerConstants.cs b/src/TriggerBinding/SqlTriggerConstants.cs index 7e1d62651..3b9cb47eb 100644 --- a/src/TriggerBinding/SqlTriggerConstants.cs +++ b/src/TriggerBinding/SqlTriggerConstants.cs @@ -32,14 +32,14 @@ internal static class SqlTriggerConstants /// /// The resource name to use for getting the application lock. We use the same resource name for all instances - /// of the function because there is some shared state across all the functions. + /// of the function because there is some shared state across all the functions. /// /// A future improvement could be to make unique application locks for each FuncId/TableId combination so that functions /// working on different tables aren't blocking each other public const string AppLockResource = "_az_func_Trigger"; /// /// Timeout for acquiring the application lock - 30sec chosen as a reasonable value to ensure we aren't - /// hanging infinitely while also giving plenty of time for the blocking transaction to complete. + /// hanging infinitely while also giving plenty of time for the blocking transaction to complete. /// public const int AppLockTimeoutMs = 30000; @@ -71,5 +71,10 @@ IF @result < 0 BEGIN RAISERROR('Unable to acquire exclusive lock on {AppLockResource}. Result = %d', 16, 1, @result) END;"; + + /// + /// There is already an object named '%.*ls' in the database. + /// + public const int ObjectAlreadyExistsErrorNumber = 2714; } } diff --git a/src/TriggerBinding/SqlTriggerListener.cs b/src/TriggerBinding/SqlTriggerListener.cs index c8d54ea3e..7786844ab 100644 --- a/src/TriggerBinding/SqlTriggerListener.cs +++ b/src/TriggerBinding/SqlTriggerListener.cs @@ -243,8 +243,14 @@ private async Task GetUserTableIdAsync(SqlConnection connection, Cancellati /// private async Task> GetPrimaryKeyColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { + const int NameIndex = 0, TypeIndex = 1, LengthIndex = 2, PrecisionIndex = 3, ScaleIndex = 4; string getPrimaryKeyColumnsQuery = $@" - SELECT c.name, t.name, c.max_length, c.precision, c.scale + SELECT + c.name, + t.name, + c.max_length, + c.precision, + c.scale FROM sys.indexes AS i INNER JOIN sys.index_columns AS ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id INNER JOIN sys.columns AS c ON ic.object_id = c.object_id AND ic.column_id = c.column_id @@ -262,20 +268,20 @@ FROM sys.indexes AS i while (await reader.ReadAsync(cancellationToken)) { - string name = reader.GetString(0); - string type = reader.GetString(1); + string name = reader.GetString(NameIndex); + string type = reader.GetString(TypeIndex); if (variableLengthTypes.Contains(type, StringComparer.OrdinalIgnoreCase)) { // Special "max" case. I'm actually not sure it's valid to have varchar(max) as a primary key because // it exceeds the byte limit of an index field (900 bytes), but just in case - short length = reader.GetInt16(2); + short length = reader.GetInt16(LengthIndex); type += length == -1 ? "(max)" : $"({length})"; } else if (variablePrecisionTypes.Contains(type)) { - byte precision = reader.GetByte(3); - byte scale = reader.GetByte(4); + byte precision = reader.GetByte(PrecisionIndex); + byte scale = reader.GetByte(ScaleIndex); type += $"({precision},{scale})"; } @@ -297,8 +303,12 @@ FROM sys.indexes AS i /// private async Task> GetUserTableColumnsAsync(SqlConnection connection, int userTableId, CancellationToken cancellationToken) { + const int NameIndex = 0, TypeIndex = 1, IsAssemblyTypeIndex = 2; string getUserTableColumnsQuery = $@" - SELECT c.name, t.name, t.is_assembly_type + SELECT + c.name, + t.name, + t.is_assembly_type FROM sys.columns AS c INNER JOIN sys.types AS t ON c.user_type_id = t.user_type_id WHERE c.object_id = {userTableId}; @@ -313,9 +323,9 @@ FROM sys.columns AS c while (await reader.ReadAsync(cancellationToken)) { - string columnName = reader.GetString(0); - string columnType = reader.GetString(1); - bool isAssemblyType = reader.GetBoolean(2); + string columnName = reader.GetString(NameIndex); + string columnType = reader.GetString(TypeIndex); + bool isAssemblyType = reader.GetBoolean(IsAssemblyTypeIndex); userTableColumns.Add(columnName); @@ -374,12 +384,11 @@ IF SCHEMA_ID(N'{SchemaName}') IS NULL { TelemetryInstance.TrackException(TelemetryErrorName.CreateSchema, ex, this._telemetryProps); var sqlEx = ex as SqlException; - if (sqlEx?.Number == 2714) + if (sqlEx?.Number == ObjectAlreadyExistsErrorNumber) { - // Error 2714 is for an object of that name already existing in the database. This generally shouldn't happen - // since we check for its existence in the statement but occasionally a race condition can make it so - // that multiple instances will try and create the schema at once. In that case we can just ignore the - // error since all we care about is that the schema exists at all. + // This generally shouldn't happen since we check for its existence in the statement but occasionally + // a race condition can make it so that multiple instances will try and create the schema at once. + // In that case we can just ignore the error since all we care about is that the schema exists at all. this._logger.LogWarning($"Failed to create schema '{SchemaName}'. Exception message: {ex.Message} This is informational only, function startup will continue as normal."); } else @@ -427,12 +436,11 @@ PRIMARY KEY (UserFunctionID, UserTableID) { TelemetryInstance.TrackException(TelemetryErrorName.CreateGlobalStateTable, ex, this._telemetryProps); var sqlEx = ex as SqlException; - if (sqlEx?.Number == 2714) + if (sqlEx?.Number == ObjectAlreadyExistsErrorNumber) { - // Error 2714 is for an object of that name already existing in the database. This generally shouldn't happen - // since we check for its existence in the statement but occasionally a race condition can make it so - // that multiple instances will try and create the table at once. In that case we can just ignore the - // error since all we care about is that the table exists at all. + // This generally shouldn't happen since we check for its existence in the statement but occasionally + // a race condition can make it so that multiple instances will try and create the schema at once. + // In that case we can just ignore the error since all we care about is that the schema exists at all. this._logger.LogWarning($"Failed to create global state table '{GlobalStateTableName}'. Exception message: {ex.Message} This is informational only, function startup will continue as normal."); } else @@ -545,12 +553,11 @@ PRIMARY KEY ({primaryKeys}) { TelemetryInstance.TrackException(TelemetryErrorName.CreateLeasesTable, ex, this._telemetryProps); var sqlEx = ex as SqlException; - if (sqlEx?.Number == 2714) + if (sqlEx?.Number == ObjectAlreadyExistsErrorNumber) { - // Error 2714 is for an object of that name already existing in the database. This generally shouldn't happen - // since we check for its existence in the statement but occasionally a race condition can make it so - // that multiple instances will try and create the table at once. In that case we can just ignore the - // error since all we care about is that the table exists at all. + // This generally shouldn't happen since we check for its existence in the statement but occasionally + // a race condition can make it so that multiple instances will try and create the schema at once. + // In that case we can just ignore the error since all we care about is that the schema exists at all. this._logger.LogWarning($"Failed to create global state table '{leasesTableName}'. Exception message: {ex.Message} This is informational only, function startup will continue as normal."); } else From 9e2cd97db7e5359d91fdf3ad342efb16b0e28ae5 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 8 Nov 2022 11:46:20 -0800 Subject: [PATCH 76/77] Update triggerbinding packages to align with main (#468) --- Directory.Packages.props | 6 +- performance/packages.lock.json | 246 +++++++++++----------- samples/samples-csharp/packages.lock.json | 238 ++++++++++----------- src/packages.lock.json | 22 +- test/packages.lock.json | 242 ++++++++++----------- 5 files changed, 377 insertions(+), 377 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index c5b581551..3945520c6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,9 +1,9 @@ - + - + @@ -15,7 +15,7 @@ - + diff --git a/performance/packages.lock.json b/performance/packages.lock.json index 9266bc760..cf3686792 100644 --- a/performance/packages.lock.json +++ b/performance/packages.lock.json @@ -114,8 +114,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.4", - "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", + "resolved": "5.2.8", + "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -123,59 +123,59 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", + "resolved": "2.2.0", + "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", + "resolved": "2.2.0", + "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", + "resolved": "2.2.0", + "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", + "resolved": "2.2.0", + "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Authorization": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Authorization": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", + "resolved": "2.2.0", + "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", + "resolved": "2.2.0", + "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Features": "2.2.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { @@ -189,12 +189,12 @@ }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", + "resolved": "2.2.0", + "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Buffers": "4.5.0" } }, @@ -208,8 +208,8 @@ }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", + "resolved": "2.2.0", + "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -217,80 +217,81 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", + "resolved": "2.2.0", + "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.1.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Extensions.DependencyInjection": "2.1.0", + "resolved": "2.2.0", + "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.2.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection": "2.2.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.0" + "System.Threading.Tasks.Extensions": "4.5.1" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", + "resolved": "2.2.0", + "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.1.0", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0" + "Microsoft.AspNetCore.JsonPatch": "2.2.0", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", + "resolved": "2.2.0", + "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.WebUtilities": "2.1.0" + "Microsoft.AspNet.WebApi.Client": "5.2.6", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", + "resolved": "2.2.0", + "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", + "resolved": "2.2.2", + "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.ObjectPool": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", + "resolved": "2.2.0", + "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.WebUtilities": { @@ -328,15 +329,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.0.2", - "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", + "resolved": "3.2.0", + "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Azure.WebJobs": "3.0.2" + "Microsoft.AspNet.WebApi.Client": "5.2.8", + "Microsoft.AspNetCore.Http": "2.2.2", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.2", + "Microsoft.Azure.WebJobs": "3.0.32" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -506,10 +507,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", + "resolved": "2.2.0", + "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -549,10 +550,10 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { @@ -574,10 +575,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", + "resolved": "2.2.0", + "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -608,13 +609,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", + "resolved": "2.2.0", + "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0" } }, "Microsoft.Extensions.Logging": { @@ -630,8 +631,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" + "resolved": "2.2.0", + "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -2102,11 +2103,11 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.14.0, )", + "Microsoft.ApplicationInsights": "[2.17.0, )", "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs": "[3.0.32, )", "Microsoft.Data.SqlClient": "[5.0.1, )", - "Newtonsoft.Json": "[11.0.2, )", + "Newtonsoft.Json": "[13.0.1, )", "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } @@ -2117,8 +2118,8 @@ "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )", - "Microsoft.NET.Sdk.Functions": "[4.1.1, )", - "Newtonsoft.Json": "[11.0.2, )" + "Microsoft.NET.Sdk.Functions": "[4.1.3, )", + "Newtonsoft.Json": "[13.0.1, )" } }, "microsoft.azure.webjobs.extensions.sql.tests": { @@ -2127,22 +2128,21 @@ "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Sql.Samples": "[1.0.0, )", - "Microsoft.NET.Sdk.Functions": "[4.1.1, )", + "Microsoft.NET.Sdk.Functions": "[4.1.3, )", "Microsoft.NET.Test.Sdk": "[17.0.0, )", "Moq": "[4.14.3, )", - "Newtonsoft.Json": "[11.0.2, )", + "Newtonsoft.Json": "[13.0.1, )", "xunit": "[2.4.0, )", "xunit.runner.visualstudio": "[2.4.0, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.14.0, )", - "resolved": "2.14.0", - "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", + "requested": "[2.17.0, )", + "resolved": "2.17.0", + "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4" + "System.Diagnostics.DiagnosticSource": "5.0.0" } }, "Microsoft.AspNetCore.Http": { @@ -2224,16 +2224,16 @@ }, "Microsoft.NET.Sdk.Functions": { "type": "CentralTransitive", - "requested": "[4.1.1, )", - "resolved": "4.1.1", - "contentHash": "G+b+bHtta7P8KPItSWygbVwQhwamU/WWmBoiv3snf8ScVYai3PbC1JSW3H22X+askqxhiw/Tx0yZKdE5oKMhRQ==", + "requested": "[4.1.3, )", + "resolved": "4.1.3", + "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", - "Newtonsoft.Json": "11.0.2" + "Newtonsoft.Json": "13.0.1" } }, "Microsoft.NET.Test.Sdk": { @@ -2264,9 +2264,9 @@ }, "Newtonsoft.Json": { "type": "CentralTransitive", - "requested": "[11.0.2, )", - "resolved": "11.0.2", - "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" }, "System.Drawing.Common": { "type": "CentralTransitive", diff --git a/samples/samples-csharp/packages.lock.json b/samples/samples-csharp/packages.lock.json index fbf18e49f..6cc85cff0 100644 --- a/samples/samples-csharp/packages.lock.json +++ b/samples/samples-csharp/packages.lock.json @@ -27,23 +27,23 @@ }, "Microsoft.NET.Sdk.Functions": { "type": "Direct", - "requested": "[4.1.1, )", - "resolved": "4.1.1", - "contentHash": "G+b+bHtta7P8KPItSWygbVwQhwamU/WWmBoiv3snf8ScVYai3PbC1JSW3H22X+askqxhiw/Tx0yZKdE5oKMhRQ==", + "requested": "[4.1.3, )", + "resolved": "4.1.3", + "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", - "Newtonsoft.Json": "11.0.2" + "Newtonsoft.Json": "13.0.1" } }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[11.0.2, )", - "resolved": "11.0.2", - "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" }, "Azure.Core": { "type": "Transitive", @@ -102,8 +102,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.4", - "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", + "resolved": "5.2.8", + "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -111,59 +111,59 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", + "resolved": "2.2.0", + "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", + "resolved": "2.2.0", + "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", + "resolved": "2.2.0", + "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", + "resolved": "2.2.0", + "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Authorization": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Authorization": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", + "resolved": "2.2.0", + "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", + "resolved": "2.2.0", + "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Features": "2.2.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { @@ -177,12 +177,12 @@ }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", + "resolved": "2.2.0", + "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Buffers": "4.5.0" } }, @@ -196,8 +196,8 @@ }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", + "resolved": "2.2.0", + "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -205,80 +205,81 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", + "resolved": "2.2.0", + "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.1.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Extensions.DependencyInjection": "2.1.0", + "resolved": "2.2.0", + "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.2.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection": "2.2.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.0" + "System.Threading.Tasks.Extensions": "4.5.1" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", + "resolved": "2.2.0", + "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.1.0", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0" + "Microsoft.AspNetCore.JsonPatch": "2.2.0", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", + "resolved": "2.2.0", + "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.WebUtilities": "2.1.0" + "Microsoft.AspNet.WebApi.Client": "5.2.6", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", + "resolved": "2.2.0", + "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", + "resolved": "2.2.2", + "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.ObjectPool": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", + "resolved": "2.2.0", + "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.WebUtilities": { @@ -316,15 +317,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.0.2", - "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", + "resolved": "3.2.0", + "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Azure.WebJobs": "3.0.2" + "Microsoft.AspNet.WebApi.Client": "5.2.8", + "Microsoft.AspNetCore.Http": "2.2.2", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.2", + "Microsoft.Azure.WebJobs": "3.0.32" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -411,10 +412,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", + "resolved": "2.2.0", + "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -454,10 +455,10 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { @@ -479,10 +480,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", + "resolved": "2.2.0", + "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -513,13 +514,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", + "resolved": "2.2.0", + "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0" } }, "Microsoft.Extensions.Logging": { @@ -535,8 +536,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" + "resolved": "2.2.0", + "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -1741,23 +1742,22 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.14.0, )", + "Microsoft.ApplicationInsights": "[2.17.0, )", "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs": "[3.0.32, )", "Microsoft.Data.SqlClient": "[5.0.1, )", - "Newtonsoft.Json": "[11.0.2, )", + "Newtonsoft.Json": "[13.0.1, )", "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.14.0, )", - "resolved": "2.14.0", - "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", + "requested": "[2.17.0, )", + "resolved": "2.17.0", + "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4" + "System.Diagnostics.DiagnosticSource": "5.0.0" } }, "Microsoft.Azure.WebJobs": { diff --git a/src/packages.lock.json b/src/packages.lock.json index ea176fd93..220fd768e 100644 --- a/src/packages.lock.json +++ b/src/packages.lock.json @@ -4,12 +4,11 @@ ".NETStandard,Version=v2.0": { "Microsoft.ApplicationInsights": { "type": "Direct", - "requested": "[2.14.0, )", - "resolved": "2.14.0", - "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", + "requested": "[2.17.0, )", + "resolved": "2.17.0", + "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4" + "System.Diagnostics.DiagnosticSource": "5.0.0" } }, "Microsoft.AspNetCore.Http": { @@ -97,9 +96,9 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[11.0.2, )", - "resolved": "11.0.2", - "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" }, "System.Runtime.Caching": { "type": "Direct", @@ -717,10 +716,11 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "4.6.0", - "contentHash": "mbBgoR0rRfl2uimsZ2avZY8g7Xnh1Mza0rJZLPcxqiMWlkGukjmRkuMJ/er+AhQuiRIh80CR/Hpeztr80seV5g==", + "resolved": "5.0.0", + "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==", "dependencies": { - "System.Memory": "4.5.3" + "System.Memory": "4.5.4", + "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, "System.Diagnostics.Process": { diff --git a/test/packages.lock.json b/test/packages.lock.json index bdef10366..898fc594c 100644 --- a/test/packages.lock.json +++ b/test/packages.lock.json @@ -17,16 +17,16 @@ }, "Microsoft.NET.Sdk.Functions": { "type": "Direct", - "requested": "[4.1.1, )", - "resolved": "4.1.1", - "contentHash": "G+b+bHtta7P8KPItSWygbVwQhwamU/WWmBoiv3snf8ScVYai3PbC1JSW3H22X+askqxhiw/Tx0yZKdE5oKMhRQ==", + "requested": "[4.1.3, )", + "resolved": "4.1.3", + "contentHash": "vpIoJxjvesBn7YOTDLLajYzlpu0DnuhV3qK+phPJ3Ywv62RwWdvqruFvZ2NtoUU8/Ad32mdhYWC3PcpuWPuyZw==", "dependencies": { "Microsoft.Azure.Functions.Analyzers": "[1.0.0, 2.0.0)", - "Microsoft.Azure.WebJobs": "[3.0.23, 3.1.0)", + "Microsoft.Azure.WebJobs": "[3.0.32, 3.1.0)", "Microsoft.Azure.WebJobs.Extensions": "3.0.6", - "Microsoft.Azure.WebJobs.Extensions.Http": "[3.0.2, 3.1.0)", + "Microsoft.Azure.WebJobs.Extensions.Http": "[3.2.0, 3.3.0)", "Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator": "4.0.1", - "Newtonsoft.Json": "11.0.2" + "Newtonsoft.Json": "13.0.1" } }, "Microsoft.NET.Test.Sdk": { @@ -51,9 +51,9 @@ }, "Newtonsoft.Json": { "type": "Direct", - "requested": "[11.0.2, )", - "resolved": "11.0.2", - "contentHash": "IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==" + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" }, "xunit": { "type": "Direct", @@ -149,8 +149,8 @@ }, "Microsoft.AspNet.WebApi.Client": { "type": "Transitive", - "resolved": "5.2.4", - "contentHash": "OdBVC2bQWkf9qDd7Mt07ev4SwIdu6VmLBMTWC0D5cOP/HWSXyv/77otwtXVrAo42duNjvXOjzjP5oOI9m1+DTQ==", + "resolved": "5.2.8", + "contentHash": "dkGLm30CxLieMlUO+oQpJw77rDs0IIx/w3lIHsp+8X94HXCsUfLYuFmlZPsQDItC0O2l1ZlWeKLkZX7ZiNRekw==", "dependencies": { "Newtonsoft.Json": "10.0.1", "Newtonsoft.Json.Bson": "1.0.1" @@ -158,59 +158,59 @@ }, "Microsoft.AspNetCore.Authentication.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "7hfl2DQoATexr0OVw8PwJSNqnu9gsbSkuHkwmHdss5xXCuY2nIfsTjj2NoKeGtp6N94ECioAP78FUfFOMj+TTg==", + "resolved": "2.2.0", + "contentHash": "VloMLDJMf3n/9ic5lCBOa42IBYJgyB1JhzLsL68Zqg+2bEPWfGBj/xCJy/LrKTArN0coOcZp3wyVTZlx0y9pHQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authentication.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NKbmBzPW2zTaZLNKkCIL7LMpr4XfXVOPJ5SNzikTe2PX3juLkupb/5oTF45wiw5srUbU6QD0cY9u3jgYUELwnQ==", + "resolved": "2.2.0", + "contentHash": "XlVJzJ5wPOYW+Y0J6Q/LVTEyfS4ssLXmt60T0SPP+D8abVhBTl+cgw2gDHlyKYIkcJg7btMVh383NDkMVqD/fg==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0" } }, "Microsoft.AspNetCore.Authorization": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "QUMtMVY7mQeJWlP8wmmhZf1HEGM/V8prW/XnYeKDpEniNBCRw0a3qktRb9aBU0vR+bpJwWZ0ibcB8QOvZEmDHQ==", + "resolved": "2.2.0", + "contentHash": "/L0W8H3jMYWyaeA9gBJqS/tSWBegP9aaTM0mjRhxTttBY9z4RVDRYJ2CwPAmAXIuPr3r1sOw+CS8jFVRGHRezQ==", "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Authorization.Policy": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "e/wxbmwHza+Y6hmM/xiQdsVX5Xh0cPHFbDTGR3kIK7a+jyBSc8CPAJOA5g0ziikLEp5Cm/Qux+CsWad53QoNOw==", + "resolved": "2.2.0", + "contentHash": "aJCo6niDRKuNg2uS2WMEmhJTooQUGARhV2ENQ2tO5443zVHUo19MSgrgGo9FIrfD+4yKPF8Q+FF33WkWfPbyKw==", "dependencies": { - "Microsoft.AspNetCore.Authentication.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Authorization": "2.1.0" + "Microsoft.AspNetCore.Authentication.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Authorization": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "1TQgBfd/NPZLR2o/h6l5Cml2ZCF5hsyV4h9WEwWwAIavrbdTnaNozGGcTOd4AOgQvogMM9UM1ajflm9Cwd0jLQ==", + "resolved": "2.2.0", + "contentHash": "ubycklv+ZY7Kutdwuy1W4upWcZ6VFR8WUXU7l7B2+mvbDBBPAcfpi+E+Y5GFe+Q157YfA3C49D2GCjAZc7Mobw==", "dependencies": { - "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.Hosting.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Hosting.Server.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.Hosting.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Hosting.Server.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "YTKMi2vHX6P+WHEVpW/DS+eFHnwivCSMklkyamcK1ETtc/4j8H3VR0kgW8XIBqukNxhD8k5wYt22P7PhrWSXjQ==", + "resolved": "2.2.0", + "contentHash": "1PMijw8RMtuQF60SsD/JlKtVfvh4NORAhF4wjysdABhlhTrYmtgssqyncR0Stq5vqtjplZcj6kbT4LRTglt9IQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Features": "2.1.0", - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Features": "2.2.0", + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.Http.Abstractions": { @@ -224,12 +224,12 @@ }, "Microsoft.AspNetCore.Http.Extensions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "M8Gk5qrUu5nFV7yE3SZgATt/5B1a5Qs8ZnXXeO/Pqu68CEiBHJWc10sdGdO5guc3zOFdm7H966mVnpZtEX4vSA==", + "resolved": "2.2.0", + "contentHash": "2DgZ9rWrJtuR7RYiew01nGRzuQBDaGHGmK56Rk54vsLLsCdzuFUPqbDTJCS1qJQWTbmbIQ9wGIOjpxA1t0l7/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0", + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0", "System.Buffers": "4.5.0" } }, @@ -243,8 +243,8 @@ }, "Microsoft.AspNetCore.JsonPatch": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "JE5LRurYn0rglbY/Nj3sB1a+yGPacyYHsuLRgvZtmjLG73R0zEfSIjGmzwtIym0HDLX0RIym8q+BLH4w1nWdog==", + "resolved": "2.2.0", + "contentHash": "o9BB9hftnCsyJalz9IT0DUFxz8Xvgh3TOfGWolpuf19duxB4FySq7c25XDYBmBMS+sun5/PsEUAi58ra4iJAoA==", "dependencies": { "Microsoft.CSharp": "4.5.0", "Newtonsoft.Json": "11.0.2" @@ -252,80 +252,81 @@ }, "Microsoft.AspNetCore.Mvc.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "NhocJc6vRjxjM8opxpbjYhdN7WbsW07eT5hZOzv87bPxwEL98Hw+D+JIu9DsPm0ce7Rao1qN1BP7w8GMhRFH0Q==", + "resolved": "2.2.0", + "contentHash": "ET6uZpfVbGR1NjCuLaLy197cQ3qZUjzl7EG5SL4GfJH/c9KRE89MMBrQegqWsh0w1iRUB/zQaK0anAjxa/pz4g==", "dependencies": { - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Net.Http.Headers": "2.1.0" + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Net.Http.Headers": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.Core": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "AtNtFLtFgZglupwiRK/9ksFg1xAXyZ1otmKtsNSFn9lIwHCQd1xZHIph7GTZiXVWn51jmauIUTUMSWdpaJ+f+A==", - "dependencies": { - "Microsoft.AspNetCore.Authentication.Core": "2.1.0", - "Microsoft.AspNetCore.Authorization.Policy": "2.1.0", - "Microsoft.AspNetCore.Hosting.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Mvc.Abstractions": "2.1.0", - "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Extensions.DependencyInjection": "2.1.0", + "resolved": "2.2.0", + "contentHash": "ALiY4a6BYsghw8PT5+VU593Kqp911U3w9f/dH9/ZoI3ezDsDAGiObqPu/HP1oXK80Ceu0XdQ3F0bx5AXBeuN/Q==", + "dependencies": { + "Microsoft.AspNetCore.Authentication.Core": "2.2.0", + "Microsoft.AspNetCore.Authorization.Policy": "2.2.0", + "Microsoft.AspNetCore.Hosting.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Http": "2.2.0", + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Mvc.Abstractions": "2.2.0", + "Microsoft.AspNetCore.ResponseCaching.Abstractions": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection": "2.2.0", "Microsoft.Extensions.DependencyModel": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", "System.Diagnostics.DiagnosticSource": "4.5.0", - "System.Threading.Tasks.Extensions": "4.5.0" + "System.Threading.Tasks.Extensions": "4.5.1" } }, "Microsoft.AspNetCore.Mvc.Formatters.Json": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Xkbx6LWehUL44rx0gcry+qY013m5LbAjqWfdeisdiSPx2bU/q4EdteRY+zDmO8vT3jKbWcAuvTVUf6AcPPQpTQ==", + "resolved": "2.2.0", + "contentHash": "ScWwXrkAvw6PekWUFkIr5qa9NKn4uZGRvxtt3DvtUrBYW5Iu2y4SS/vx79JN0XDHNYgAJ81nVs+4M7UE1Y/O+g==", "dependencies": { - "Microsoft.AspNetCore.JsonPatch": "2.1.0", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0" + "Microsoft.AspNetCore.JsonPatch": "2.2.0", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0" } }, "Microsoft.AspNetCore.Mvc.WebApiCompatShim": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "pYsNGveHyMCHQ+xpUIsTHtFFv7Xm+q2pmL3UmL6QujO5ICu/bcnSlwu9FEQhXYQ+cDxfO2VShdM/OrkWzNFGFw==", + "resolved": "2.2.0", + "contentHash": "YKovpp46Fgah0N8H4RGb+7x9vdjj50mS3NON910pYJFQmn20Cd1mYVkTunjy/DrZpvwmJ8o5Es0VnONSYVXEAQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Mvc.Core": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.WebUtilities": "2.1.0" + "Microsoft.AspNet.WebApi.Client": "5.2.6", + "Microsoft.AspNetCore.Mvc.Core": "2.2.0", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.WebUtilities": "2.2.0" } }, "Microsoft.AspNetCore.ResponseCaching.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "Ht/KGFWYqcUDDi+VMPkQNzY7wQ0I2SdqXMEPl6AsOW8hmO3ZS4jIPck6HGxIdlk7ftL9YITJub0cxBmnuq+6zQ==", + "resolved": "2.2.0", + "contentHash": "CIHWEKrHzZfFp7t57UXsueiSA/raku56TgRYauV/W1+KAQq6vevz60zjEKaazt3BI76zwMz3B4jGWnCwd8kwQw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.AspNetCore.Routing": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "eRdsCvtUlLsh0O2Q8JfcpTUhv0m5VCYkgjZTCdniGAq7F31B3gNrBTn9VMqz14m+ZxPUzNqudfDFVTAQlrI/5Q==", + "resolved": "2.2.2", + "contentHash": "HcmJmmGYewdNZ6Vcrr5RkQbc/YWU4F79P3uPPBi6fCFOgUewXNM1P4kbPuoem7tN4f7x8mq7gTsm5QGohQ5g/w==", "dependencies": { - "Microsoft.AspNetCore.Http.Extensions": "2.1.0", - "Microsoft.AspNetCore.Routing.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0", - "Microsoft.Extensions.ObjectPool": "2.1.0", - "Microsoft.Extensions.Options": "2.1.0" + "Microsoft.AspNetCore.Http.Extensions": "2.2.0", + "Microsoft.AspNetCore.Routing.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0", + "Microsoft.Extensions.ObjectPool": "2.2.0", + "Microsoft.Extensions.Options": "2.2.0" } }, "Microsoft.AspNetCore.Routing.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "LXmnHeb3v+HTfn74M46s+4wLaMkplj1Yl2pRf+2mfDDsQ7PN0+h8AFtgip5jpvBvFHQ/Pei7S+cSVsSTHE67fQ==", + "resolved": "2.2.0", + "contentHash": "lRRaPN7jDlUCVCp9i0W+PB0trFaKB0bgMJD7hEJS9Uo4R9MXaMC8X2tJhPLmeVE3SGDdYI4QNKdVmhNvMJGgPQ==", "dependencies": { - "Microsoft.AspNetCore.Http.Abstractions": "2.1.0" + "Microsoft.AspNetCore.Http.Abstractions": "2.2.0" } }, "Microsoft.AspNetCore.WebUtilities": { @@ -363,15 +364,15 @@ }, "Microsoft.Azure.WebJobs.Extensions.Http": { "type": "Transitive", - "resolved": "3.0.2", - "contentHash": "JvC3fESMMbNkYbpaJ4vkK4Xaw1yZy4HSxxqwoaI3Ls2Y5/qBrHftPy0WJQgmXcGjgE/o/aAmuixdTfrj5OQDJQ==", + "resolved": "3.2.0", + "contentHash": "IXLuo5fOliOYKUZjWO5kQ/j3XblM9TNnk1agjzNYkubpDXq6M436GihaVzwTeQlX279P3G1KquS6I+b7pXaFuQ==", "dependencies": { - "Microsoft.AspNet.WebApi.Client": "5.2.4", - "Microsoft.AspNetCore.Http": "2.1.0", - "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.1.0", - "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.1.0", - "Microsoft.AspNetCore.Routing": "2.1.0", - "Microsoft.Azure.WebJobs": "3.0.2" + "Microsoft.AspNet.WebApi.Client": "5.2.8", + "Microsoft.AspNetCore.Http": "2.2.2", + "Microsoft.AspNetCore.Mvc.Formatters.Json": "2.2.0", + "Microsoft.AspNetCore.Mvc.WebApiCompatShim": "2.2.0", + "Microsoft.AspNetCore.Routing": "2.2.2", + "Microsoft.Azure.WebJobs": "3.0.32" } }, "Microsoft.Azure.WebJobs.Extensions.Storage.Blobs": { @@ -463,10 +464,10 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "VfuZJNa0WUshZ/+8BFZAhwFKiKuu/qOUCFntfdLpHj7vcRnsGHqd3G2Hse78DM+pgozczGM63lGPRLmy+uhUOA==", + "resolved": "2.2.0", + "contentHash": "65MrmXCziWaQFrI0UHkQbesrX5wTwf9XPjY5yFm/VkgJKFJ5gqvXRoXjIZcf2wLi5ZlwGz/oMYfyURVCWbM5iw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.1" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.Configuration.Binder": { @@ -506,10 +507,10 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "gqQviLfuA31PheEGi+XJoZc1bc9H9RsPa9Gq9XuDct7XGWSR9eVXjK5Sg7CSUPhTFHSuxUFY12wcTYLZ4zM1hg==", + "resolved": "2.2.0", + "contentHash": "MZtBIwfDFork5vfjpJdG5g8wuJFt7d/y3LOSVVtDK/76wlbtz6cjltfKHqLx2TKVqTj5/c41t77m1+h20zqtPA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { @@ -531,10 +532,10 @@ }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "itv+7XBu58pxi8mykxx9cUO1OOVYe0jmQIZVSZVp5lOcLxB7sSV2bnHiI1RSu6Nxne/s6+oBla3ON5CCMSmwhQ==", + "resolved": "2.2.0", + "contentHash": "EcnaSsPTqx2MGnHrmWOD0ugbuuqVT8iICqSqPzi45V5/MA1LjUNb0kwgcxBGqizV1R+WeBK7/Gw25Jzkyk9bIw==", "dependencies": { - "Microsoft.Extensions.Primitives": "2.1.0" + "Microsoft.Extensions.Primitives": "2.2.0" } }, "Microsoft.Extensions.FileProviders.Physical": { @@ -565,13 +566,13 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", - "resolved": "2.1.0", - "contentHash": "BpMaoBxdXr5VD0yk7rYN6R8lAU9X9JbvsPveNdKT+llIn3J5s4sxpWqaSG/NnzTzTLU5eJE5nrecTl7clg/7dQ==", + "resolved": "2.2.0", + "contentHash": "+k4AEn68HOJat5gj1TWa6X28WlirNQO9sPIIeQbia+91n03esEtMSSoekSTpMjUzjqtJWQN3McVx0GvSPFHF/Q==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "2.1.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "2.1.0", - "Microsoft.Extensions.FileProviders.Abstractions": "2.1.0", - "Microsoft.Extensions.Logging.Abstractions": "2.1.0" + "Microsoft.Extensions.Configuration.Abstractions": "2.2.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "2.2.0", + "Microsoft.Extensions.FileProviders.Abstractions": "2.2.0", + "Microsoft.Extensions.Logging.Abstractions": "2.2.0" } }, "Microsoft.Extensions.Logging": { @@ -587,8 +588,8 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "2.1.1", - "contentHash": "XRzK7ZF+O6FzdfWrlFTi1Rgj2080ZDsd46vzOjadHUB0Cz5kOvDG8vI7caa5YFrsHQpcfn0DxtjS4E46N4FZsA==" + "resolved": "2.2.0", + "contentHash": "B2WqEox8o+4KUOpL7rZPyh6qYjik8tHi2tN8Z9jZkHzED8ElYgZa/h6K+xliB435SqUcWT290Fr2aa8BtZjn8A==" }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", @@ -1944,11 +1945,11 @@ "microsoft.azure.webjobs.extensions.sql": { "type": "Project", "dependencies": { - "Microsoft.ApplicationInsights": "[2.14.0, )", + "Microsoft.ApplicationInsights": "[2.17.0, )", "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs": "[3.0.32, )", "Microsoft.Data.SqlClient": "[5.0.1, )", - "Newtonsoft.Json": "[11.0.2, )", + "Newtonsoft.Json": "[13.0.1, )", "System.Runtime.Caching": "[5.0.0, )", "morelinq": "[3.3.2, )" } @@ -1959,18 +1960,17 @@ "Microsoft.AspNetCore.Http": "[2.2.2, )", "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )", "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )", - "Microsoft.NET.Sdk.Functions": "[4.1.1, )", - "Newtonsoft.Json": "[11.0.2, )" + "Microsoft.NET.Sdk.Functions": "[4.1.3, )", + "Newtonsoft.Json": "[13.0.1, )" } }, "Microsoft.ApplicationInsights": { "type": "CentralTransitive", - "requested": "[2.14.0, )", - "resolved": "2.14.0", - "contentHash": "j66/uh1Mb5Px/nvMwDWx73Wd9MdO2X+zsDw9JScgm5Siz5r4Cw8sTW+VCrjurFkpFqrNkj+0wbfRHDBD9b+elA==", + "requested": "[2.17.0, )", + "resolved": "2.17.0", + "contentHash": "moAOrjhwiCWdg8I4fXPEd+bnnyCSRxo6wmYQ0HuNrWJUctzZEiyVTbJ8QTS6+dBOFTxpI6x+OY5wHPHrgWOk1Q==", "dependencies": { - "System.Diagnostics.DiagnosticSource": "4.6.0", - "System.Memory": "4.5.4" + "System.Diagnostics.DiagnosticSource": "5.0.0" } }, "Microsoft.Azure.WebJobs": { From e813a4b50f90901c6b63266f74ec65b2da4ba60f Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Tue, 8 Nov 2022 11:50:10 -0800 Subject: [PATCH 77/77] Remove triggerbindings from pr builds --- builds/azure-pipelines/build-pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/builds/azure-pipelines/build-pr.yml b/builds/azure-pipelines/build-pr.yml index 4442edc01..5c4829b96 100644 --- a/builds/azure-pipelines/build-pr.yml +++ b/builds/azure-pipelines/build-pr.yml @@ -4,7 +4,6 @@ pr: branches: include: - main - - triggerbindings variables: solution: '**/*.sln'