From 3a4ddd387d0b21f48282c79db3e9e646b94d7b23 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 11:02:53 -0700 Subject: [PATCH 1/5] Fix polling size override using wrong configuration value --- src/TriggerBinding/SqlTableChangeMonitor.cs | 2 +- test/Common/TestUtils.cs | 31 +++++++++++++ .../SqlTriggerBindingIntegrationTests.cs | 43 +++++++++++++++---- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/TriggerBinding/SqlTableChangeMonitor.cs b/src/TriggerBinding/SqlTableChangeMonitor.cs index 5ed030ae3..bdfccca79 100644 --- a/src/TriggerBinding/SqlTableChangeMonitor.cs +++ b/src/TriggerBinding/SqlTableChangeMonitor.cs @@ -124,7 +124,7 @@ public SqlTableChangeMonitor( // Check if there's config settings to override the default batch size/polling interval values int? configuredBatchSize = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize); - int? configuredPollingInterval = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_BatchSize); + int? configuredPollingInterval = configuration.GetValue(SqlTriggerConstants.ConfigKey_SqlTrigger_PollingInterval); this._batchSize = configuredBatchSize ?? this._batchSize; this._pollingIntervalInMs = configuredPollingInterval ?? this._pollingIntervalInMs; var monitorStartProps = new Dictionary(telemetryProps) diff --git a/test/Common/TestUtils.cs b/test/Common/TestUtils.cs index ab45410a3..7cef4e51d 100644 --- a/test/Common/TestUtils.cs +++ b/test/Common/TestUtils.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Data; using System.Diagnostics; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -177,5 +178,35 @@ public static async Task TimeoutAfter(this Task task, throw new TimeoutException(message); } } + + /// + /// Creates a DataReceievedEventHandler that will wait for the specified regex and then check that + /// the matched group matches the expected value. + /// + /// The task completion source to signal when the value is receieved + /// The regex. This must have a single group match for the specific value being looked for + /// The name of the value to output if the match fails + /// The value expected to be equal to the matched group from the regex + /// + public static DataReceivedEventHandler CreateOutputReceievedHandler(TaskCompletionSource taskCompletionSource, string regex, string valueName, string expectedValue) + { + return (object sender, DataReceivedEventArgs e) => + { + Match match = Regex.Match(e.Data, regex); + if (match.Success) + { + // We found the line so now check that the group matches our expected value + string actualValue = match.Groups[1].Value; + if (actualValue == expectedValue) + { + taskCompletionSource.SetResult(true); + } + else + { + taskCompletionSource.SetException(new Exception($"Expected {valueName} value of {expectedValue} but got value {actualValue}")); + } + } + }; + } } } diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index baca890af..a8c3ab4a7 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -79,10 +79,22 @@ public async Task BatchSizeOverrideTriggerTest() const int firstId = 1; const int lastId = 40; this.EnableChangeTrackingForTable("Products"); - this.StartFunctionHost(nameof(ProductsTriggerWithValidation), SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { - { "TEST_EXPECTED_BATCH_SIZE", batchSize.ToString() }, - { "Sql_Trigger_BatchSize", batchSize.ToString() } - }); + var taskCompletionSource = new TaskCompletionSource(); + DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( + taskCompletionSource, + @"Starting change consumption loop. BatchSize: \d* PollingIntervalMs: (\d*)", + "PollingInterval", + batchSize.ToString()); + this.StartFunctionHost( + nameof(ProductsTriggerWithValidation), + SupportedLanguages.CSharp, + useTestFolder: true, + customOutputHandler: handler, + environmentVariables: new Dictionary() { + { "TEST_EXPECTED_BATCH_SIZE", batchSize.ToString() }, + { "Sql_Trigger_BatchSize", batchSize.ToString() } + } + ); await this.WaitForProductChanges( firstId, @@ -92,6 +104,7 @@ await this.WaitForProductChanges( id => $"Product {id}", id => id * 100, this.GetBatchProcessingTimeout(firstId, lastId, batchSize: batchSize)); + await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000)); } /// @@ -100,13 +113,25 @@ await this.WaitForProductChanges( [Fact] public async Task PollingIntervalOverrideTriggerTest() { - const int pollingIntervalMs = 100; const int firstId = 1; const int lastId = 50; + const int pollingIntervalMs = 75; this.EnableChangeTrackingForTable("Products"); - this.StartFunctionHost(nameof(ProductsTriggerWithValidation), SupportedLanguages.CSharp, true, environmentVariables: new Dictionary() { - { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } - }); + var taskCompletionSource = new TaskCompletionSource(); + DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( + taskCompletionSource, + @"Starting change consumption loop. BatchSize: \d* PollingIntervalMs: (\d*)", + "PollingInterval", + pollingIntervalMs.ToString()); + this.StartFunctionHost( + nameof(ProductsTriggerWithValidation), + SupportedLanguages.CSharp, + useTestFolder: true, + customOutputHandler: handler, + environmentVariables: new Dictionary() { + { "Sql_Trigger_PollingIntervalMs", pollingIntervalMs.ToString() } + } + ); await this.WaitForProductChanges( firstId, @@ -116,9 +141,9 @@ await this.WaitForProductChanges( id => $"Product {id}", id => id * 100, this.GetBatchProcessingTimeout(firstId, lastId, pollingIntervalMs: pollingIntervalMs)); + await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000)); } - /// /// Verifies that if several changes have happened to the table row since last invocation, then a single net /// change for that row is passed to the user function. From 2e0f3454a282d6d67b0f3aed335b366a931434cd Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 11:22:07 -0700 Subject: [PATCH 2/5] Fixes --- .../SqlTriggerBindingIntegrationTests.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index a8c3ab4a7..0579f1527 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -75,15 +75,19 @@ await this.WaitForProductChanges( [Fact] public async Task BatchSizeOverrideTriggerTest() { - const int batchSize = 20; + // Use enough items for the default batch size to require 4 batches but then + // set the batch size to the same value so they can all be processed in one + // batch. The test will only wait for ~1 batch worth of time so will timeout + // if the batch size isn't actually changed + const int batchSize = SqlTableChangeMonitor.DefaultBatchSize * 4; const int firstId = 1; - const int lastId = 40; + const int lastId = batchSize; this.EnableChangeTrackingForTable("Products"); var taskCompletionSource = new TaskCompletionSource(); DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( taskCompletionSource, - @"Starting change consumption loop. BatchSize: \d* PollingIntervalMs: (\d*)", - "PollingInterval", + @"Starting change consumption loop. BatchSize: (\d*) PollingIntervalMs: \d*", + "BatchSize", batchSize.ToString()); this.StartFunctionHost( nameof(ProductsTriggerWithValidation), @@ -114,7 +118,10 @@ await this.WaitForProductChanges( public async Task PollingIntervalOverrideTriggerTest() { const int firstId = 1; - const int lastId = 50; + // Use enough items to require 5 batches to be processed - the test will + // only wait for the expected time and timeout if the default polling + // interval isn't actually modified. + const int lastId = SqlTableChangeMonitor.DefaultBatchSize * 5; const int pollingIntervalMs = 75; this.EnableChangeTrackingForTable("Products"); var taskCompletionSource = new TaskCompletionSource(); From 360587c291c3909db6e3e0577c898b06931c4174 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 12:07:48 -0700 Subject: [PATCH 3/5] Update comments --- test/Common/TestUtils.cs | 2 +- test/Integration/SqlTriggerBindingIntegrationTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Common/TestUtils.cs b/test/Common/TestUtils.cs index 7cef4e51d..ae0d0f657 100644 --- a/test/Common/TestUtils.cs +++ b/test/Common/TestUtils.cs @@ -187,7 +187,7 @@ public static async Task TimeoutAfter(this Task task, /// The regex. This must have a single group match for the specific value being looked for /// The name of the value to output if the match fails /// The value expected to be equal to the matched group from the regex - /// + /// The event handler public static DataReceivedEventHandler CreateOutputReceievedHandler(TaskCompletionSource taskCompletionSource, string regex, string valueName, string expectedValue) { return (object sender, DataReceivedEventArgs e) => diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 0579f1527..7fbb9a81c 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -75,7 +75,7 @@ await this.WaitForProductChanges( [Fact] public async Task BatchSizeOverrideTriggerTest() { - // Use enough items for the default batch size to require 4 batches but then + // Use enough items to require 4 batches to be processed but then // set the batch size to the same value so they can all be processed in one // batch. The test will only wait for ~1 batch worth of time so will timeout // if the batch size isn't actually changed From 42d301e8edde7488c39e7d1a6a33283e6a73006d Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 12:12:32 -0700 Subject: [PATCH 4/5] typo --- test/Common/TestUtils.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Common/TestUtils.cs b/test/Common/TestUtils.cs index ae0d0f657..8ba874b9f 100644 --- a/test/Common/TestUtils.cs +++ b/test/Common/TestUtils.cs @@ -183,7 +183,7 @@ public static async Task TimeoutAfter(this Task task, /// Creates a DataReceievedEventHandler that will wait for the specified regex and then check that /// the matched group matches the expected value. /// - /// The task completion source to signal when the value is receieved + /// The task completion source to signal when the value is received /// The regex. This must have a single group match for the specific value being looked for /// The name of the value to output if the match fails /// The value expected to be equal to the matched group from the regex From 450180e43c9b1514be3786425898963ab89f0947 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Fri, 14 Oct 2022 14:26:53 -0700 Subject: [PATCH 5/5] more changes --- test/Integration/SqlTriggerBindingIntegrationTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Integration/SqlTriggerBindingIntegrationTests.cs b/test/Integration/SqlTriggerBindingIntegrationTests.cs index 7fbb9a81c..a1f6d4c80 100644 --- a/test/Integration/SqlTriggerBindingIntegrationTests.cs +++ b/test/Integration/SqlTriggerBindingIntegrationTests.cs @@ -108,7 +108,7 @@ await this.WaitForProductChanges( id => $"Product {id}", id => id * 100, this.GetBatchProcessingTimeout(firstId, lastId, batchSize: batchSize)); - await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000)); + await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000), "Timed out waiting for BatchSize configuration message"); } /// @@ -122,7 +122,7 @@ public async Task PollingIntervalOverrideTriggerTest() // only wait for the expected time and timeout if the default polling // interval isn't actually modified. const int lastId = SqlTableChangeMonitor.DefaultBatchSize * 5; - const int pollingIntervalMs = 75; + const int pollingIntervalMs = SqlTableChangeMonitor.DefaultPollingIntervalMs / 2; this.EnableChangeTrackingForTable("Products"); var taskCompletionSource = new TaskCompletionSource(); DataReceivedEventHandler handler = TestUtils.CreateOutputReceievedHandler( @@ -148,7 +148,7 @@ await this.WaitForProductChanges( id => $"Product {id}", id => id * 100, this.GetBatchProcessingTimeout(firstId, lastId, pollingIntervalMs: pollingIntervalMs)); - await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000)); + await taskCompletionSource.Task.TimeoutAfter(TimeSpan.FromSeconds(5000), "Timed out waiting for PollingInterval configuration message"); } ///