From 3a9ef288231dbc572f639e4e2ab75c6b1d057b39 Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 09:08:16 -0700 Subject: [PATCH 01/15] cp sql dll to extension bundle --- builds/azure-pipelines/template-steps-build-test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 40b6b9165..e56c94490 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -32,6 +32,9 @@ steps: displayName: 'Set npm installation path for Windows' condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) +- bash: echo "##vso[task.setvariable variable=azureFunctionsExtensionBundlePath]$(func GetExtensionBundlePath)" + displayName: 'Set Azure Functions extension bundle path' + - task: DockerInstaller@0 displayName: Docker Installer inputs: @@ -67,6 +70,12 @@ steps: projects: '${{ parameters.solution }}' arguments: '--configuration ${{ parameters.configuration }} -p:GeneratePackageOnBuild=false -p:Version=${{ parameters.binariesVersion }}' +- task: CopyFiles@2 + displayName: 'Copy Sql extension dll to Azure Functions extension bundle' + inputs: + contents: '$(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\Microsoft.Azure.WebJobs.Extensions.Sql.dll' + targetFolder: '$(azureFunctionsExtensionBundlePath)' + - script: | npm install npm run lint From e303bc1d3b66b8fb05c3af86a80914b007753fb2 Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 11:57:40 -0700 Subject: [PATCH 02/15] remove quotes + add windows condition --- builds/azure-pipelines/template-steps-build-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index e56c94490..e7f4b900b 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -73,8 +73,9 @@ steps: - task: CopyFiles@2 displayName: 'Copy Sql extension dll to Azure Functions extension bundle' inputs: - contents: '$(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\Microsoft.Azure.WebJobs.Extensions.Sql.dll' - targetFolder: '$(azureFunctionsExtensionBundlePath)' + contents: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\Microsoft.Azure.WebJobs.Extensions.Sql.dll + targetFolder: $(azureFunctionsExtensionBundlePath) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - script: | npm install From 19ae5ecb7d6275e44293e6543ffa43de5e488d93 Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 12:17:00 -0700 Subject: [PATCH 03/15] fix path --- builds/azure-pipelines/template-steps-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index e7f4b900b..046209378 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -73,7 +73,7 @@ steps: - task: CopyFiles@2 displayName: 'Copy Sql extension dll to Azure Functions extension bundle' inputs: - contents: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\Microsoft.Azure.WebJobs.Extensions.Sql.dll + contents: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\**\Microsoft.Azure.WebJobs.Extensions.Sql.dll targetFolder: $(azureFunctionsExtensionBundlePath) condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) From 242b5941e20c530403ff412d139e0efeecfb34e7 Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 12:53:14 -0700 Subject: [PATCH 04/15] add bin to path + add linux --- builds/azure-pipelines/template-steps-build-test.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 046209378..1d5af692d 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -71,12 +71,19 @@ steps: arguments: '--configuration ${{ parameters.configuration }} -p:GeneratePackageOnBuild=false -p:Version=${{ parameters.binariesVersion }}' - task: CopyFiles@2 - displayName: 'Copy Sql extension dll to Azure Functions extension bundle' + displayName: 'Copy Sql extension dll to Azure Functions extension bundle for Windows' inputs: contents: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\**\Microsoft.Azure.WebJobs.Extensions.Sql.dll - targetFolder: $(azureFunctionsExtensionBundlePath) + targetFolder: $(azureFunctionsExtensionBundlePath)\bin condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) +- task: CopyFiles@2 + displayName: 'Copy Sql extension dll to Azure Functions extension bundle for Linux' + inputs: + contents: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/**/Microsoft.Azure.WebJobs.Extensions.Sql.dll + targetFolder: $(azureFunctionsExtensionBundlePath)/bin + condition: and(succeeded(), eq(variables['Agent.OS'], 'linux')) + - script: | npm install npm run lint From f644f5a9be05462b8aefaa568e81bc4f5ed093ef Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 13:31:10 -0700 Subject: [PATCH 05/15] use source folder --- builds/azure-pipelines/template-steps-build-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 1d5af692d..242a86adf 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -73,14 +73,16 @@ steps: - task: CopyFiles@2 displayName: 'Copy Sql extension dll to Azure Functions extension bundle for Windows' inputs: - contents: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\**\Microsoft.Azure.WebJobs.Extensions.Sql.dll + sourceFolder: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\netstandard2.0 + contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll targetFolder: $(azureFunctionsExtensionBundlePath)\bin condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - task: CopyFiles@2 displayName: 'Copy Sql extension dll to Azure Functions extension bundle for Linux' inputs: - contents: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/**/Microsoft.Azure.WebJobs.Extensions.Sql.dll + sourceFolder: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/netstandard2.0 + contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll targetFolder: $(azureFunctionsExtensionBundlePath)/bin condition: and(succeeded(), eq(variables['Agent.OS'], 'linux')) From 471e9768e94f00aca056d58b58eaa9b47d0e2aa3 Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 14:22:34 -0700 Subject: [PATCH 06/15] use preview path --- builds/azure-pipelines/template-steps-build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 242a86adf..324c8df9d 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -75,7 +75,7 @@ steps: inputs: sourceFolder: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\netstandard2.0 contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll - targetFolder: $(azureFunctionsExtensionBundlePath)\bin + targetFolder: $(azureFunctionsExtensionBundlePath)\..\..\Microsoft.Azure.Functions.ExtensionBundle.Preview\**\bin condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - task: CopyFiles@2 @@ -83,7 +83,7 @@ steps: inputs: sourceFolder: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/netstandard2.0 contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll - targetFolder: $(azureFunctionsExtensionBundlePath)/bin + targetFolder: $(azureFunctionsExtensionBundlePath)/../../Microsoft.Azure.Functions.ExtensionBundle.Preview/**/bin condition: and(succeeded(), eq(variables['Agent.OS'], 'linux')) - script: | From dc9525b2cacc41ee402851f86d45c567ad3f5130 Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 15:24:58 -0700 Subject: [PATCH 07/15] add overwrite and ignore mkdir errors --- builds/azure-pipelines/template-steps-build-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 324c8df9d..28a9fdd00 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -76,6 +76,8 @@ steps: sourceFolder: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\netstandard2.0 contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll targetFolder: $(azureFunctionsExtensionBundlePath)\..\..\Microsoft.Azure.Functions.ExtensionBundle.Preview\**\bin + overWrite: true + ignoreMakeDirErrors: true condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - task: CopyFiles@2 @@ -84,6 +86,8 @@ steps: sourceFolder: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/netstandard2.0 contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll targetFolder: $(azureFunctionsExtensionBundlePath)/../../Microsoft.Azure.Functions.ExtensionBundle.Preview/**/bin + overWrite: true + ignoreMakeDirErrors: true condition: and(succeeded(), eq(variables['Agent.OS'], 'linux')) - script: | From 46a5c388794a844e23bd1f43527cf3b42fe361e5 Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 16:10:11 -0700 Subject: [PATCH 08/15] fix target folder path --- builds/azure-pipelines/template-steps-build-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 28a9fdd00..9f49b49fb 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -34,6 +34,7 @@ steps: - bash: echo "##vso[task.setvariable variable=azureFunctionsExtensionBundlePath]$(func GetExtensionBundlePath)" displayName: 'Set Azure Functions extension bundle path' + workingDirectory: $(Build.SourcesDirectory)/samples/samples-js - task: DockerInstaller@0 displayName: Docker Installer @@ -75,7 +76,7 @@ steps: inputs: sourceFolder: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\netstandard2.0 contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll - targetFolder: $(azureFunctionsExtensionBundlePath)\..\..\Microsoft.Azure.Functions.ExtensionBundle.Preview\**\bin + targetFolder: $(azureFunctionsExtensionBundlePath)\bin overWrite: true ignoreMakeDirErrors: true condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) @@ -85,7 +86,7 @@ steps: inputs: sourceFolder: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/netstandard2.0 contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll - targetFolder: $(azureFunctionsExtensionBundlePath)/../../Microsoft.Azure.Functions.ExtensionBundle.Preview/**/bin + targetFolder: $(azureFunctionsExtensionBundlePath)/bin overWrite: true ignoreMakeDirErrors: true condition: and(succeeded(), eq(variables['Agent.OS'], 'linux')) From 27705b96ce569c47c8d50afeb47dd480ae85bde7 Mon Sep 17 00:00:00 2001 From: luczhan Date: Wed, 12 Oct 2022 17:40:39 -0700 Subject: [PATCH 09/15] remove ignoreMakeDirErrors --- builds/azure-pipelines/template-steps-build-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 9f49b49fb..d21517e81 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -78,7 +78,6 @@ steps: contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll targetFolder: $(azureFunctionsExtensionBundlePath)\bin overWrite: true - ignoreMakeDirErrors: true condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - task: CopyFiles@2 @@ -88,7 +87,6 @@ steps: contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll targetFolder: $(azureFunctionsExtensionBundlePath)/bin overWrite: true - ignoreMakeDirErrors: true condition: and(succeeded(), eq(variables['Agent.OS'], 'linux')) - script: | From 4b2f0d2cabb4cc31b3ebc3a49a14947dd435442f Mon Sep 17 00:00:00 2001 From: luczhan Date: Thu, 13 Oct 2022 06:19:32 -0700 Subject: [PATCH 10/15] add wait for case sensitivity test --- test/Integration/SqlOutputBindingIntegrationTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Integration/SqlOutputBindingIntegrationTests.cs b/test/Integration/SqlOutputBindingIntegrationTests.cs index 9891cb526..4b3436ba6 100644 --- a/test/Integration/SqlOutputBindingIntegrationTests.cs +++ b/test/Integration/SqlOutputBindingIntegrationTests.cs @@ -355,7 +355,7 @@ public void AddProductCaseSensitiveTest(SupportedLanguages lang) this.StartFunctionHost(nameof(AddProductParams), lang); // Change database collation to case sensitive - this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CS_AS"); + this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} SET Single_User WITH ROLLBACK IMMEDIATE; ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CS_AS; ALTER DATABASE {this.DatabaseName} SET Multi_User;"); var query = new Dictionary() { @@ -369,7 +369,9 @@ public void AddProductCaseSensitiveTest(SupportedLanguages lang) Assert.Throws(() => this.SendOutputGetRequest("addproduct-params", query).Wait()); // Change database collation back to case insensitive - this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CI_AS"); + this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} SET Single_User WITH ROLLBACK IMMEDIATE; ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CI_AS; ALTER DATABASE {this.DatabaseName} SET Multi_User;"); + + Thread.Sleep(660000); // Wait 11 minutes to ensure table info cache is not hit since database collation was changed this.SendOutputGetRequest("addproduct-params", query).Wait(); From 3a31f313b55cba93cfe7f9482b5fa25a1973e47e Mon Sep 17 00:00:00 2001 From: luczhan Date: Thu, 13 Oct 2022 10:37:30 -0700 Subject: [PATCH 11/15] use one task --- .../azure-pipelines/template-steps-build-test.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index d21517e81..762076709 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -72,22 +72,12 @@ steps: arguments: '--configuration ${{ parameters.configuration }} -p:GeneratePackageOnBuild=false -p:Version=${{ parameters.binariesVersion }}' - task: CopyFiles@2 - displayName: 'Copy Sql extension dll to Azure Functions extension bundle for Windows' - inputs: - sourceFolder: $(Build.SourcesDirectory)\src\bin\${{ parameters.configuration }}\netstandard2.0 - contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll - targetFolder: $(azureFunctionsExtensionBundlePath)\bin - overWrite: true - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -- task: CopyFiles@2 - displayName: 'Copy Sql extension dll to Azure Functions extension bundle for Linux' + displayName: 'Copy Sql extension dll to Azure Functions extension bundle' inputs: sourceFolder: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/netstandard2.0 contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll - targetFolder: $(azureFunctionsExtensionBundlePath)/bin + targetFolder: $(azureFunctionsExtensionBundlePath)\bin overWrite: true - condition: and(succeeded(), eq(variables['Agent.OS'], 'linux')) - script: | npm install From 67adc3e2a376e7d70c0da2646716d4ef69acbae7 Mon Sep 17 00:00:00 2001 From: luczhan Date: Thu, 13 Oct 2022 11:22:44 -0700 Subject: [PATCH 12/15] set cache timeout to 0 --- .../azure-pipelines/template-steps-build-test.yml | 1 + src/SqlAsyncCollector.cs | 13 ++++++++++++- .../Integration/SqlOutputBindingIntegrationTests.cs | 2 -- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 762076709..3b88ae9ea 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -247,6 +247,7 @@ steps: TEST_SERVER: '$(testServer)' NODE_MODULES_PATH: '$(nodeModulesPath)' AZUREFUNCTIONS_SQLBINDINGS_TELEMETRY_OPTOUT: '1' + AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MIN: '0' inputs: command: test projects: '${{ parameters.solution }}' diff --git a/src/SqlAsyncCollector.cs b/src/SqlAsyncCollector.cs index ca6e420b1..18a6090f1 100644 --- a/src/SqlAsyncCollector.cs +++ b/src/SqlAsyncCollector.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Globalization; using System.Collections.Generic; using System.Data; using System.Linq; @@ -171,6 +172,16 @@ private async Task UpsertRowsAsync(IEnumerable rows, SqlAttribute attribute, ObjectCache cachedTables = MemoryCache.Default; var tableInfo = cachedTables[cacheKey] as TableInformation; + int timeout = 10; + string timeoutEnvVar = Environment.GetEnvironmentVariable("AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MIN"); + this._logger.LogDebugWithThreadId("TIMEOUT STRING HERE " + timeoutEnvVar); + + if (!string.IsNullOrEmpty(timeoutEnvVar)) + { + timeout = int.Parse(timeoutEnvVar, CultureInfo.InvariantCulture); + } + this._logger.LogDebugWithThreadId("TIMEOUT HERE " + timeout); + if (tableInfo == null) { TelemetryInstance.TrackEvent(TelemetryEventName.TableInfoCacheMiss, props); @@ -179,7 +190,7 @@ private async Task UpsertRowsAsync(IEnumerable rows, SqlAttribute attribute, var policy = new CacheItemPolicy { // Re-look up the primary key(s) after 10 minutes (they should not change very often!) - AbsoluteExpiration = DateTimeOffset.Now.AddMinutes(10) + AbsoluteExpiration = DateTimeOffset.Now.AddMinutes(timeout) }; cachedTables.Set(cacheKey, tableInfo, policy); diff --git a/test/Integration/SqlOutputBindingIntegrationTests.cs b/test/Integration/SqlOutputBindingIntegrationTests.cs index 4b3436ba6..c2415849c 100644 --- a/test/Integration/SqlOutputBindingIntegrationTests.cs +++ b/test/Integration/SqlOutputBindingIntegrationTests.cs @@ -371,8 +371,6 @@ public void AddProductCaseSensitiveTest(SupportedLanguages lang) // Change database collation back to case insensitive this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} SET Single_User WITH ROLLBACK IMMEDIATE; ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CI_AS; ALTER DATABASE {this.DatabaseName} SET Multi_User;"); - Thread.Sleep(660000); // Wait 11 minutes to ensure table info cache is not hit since database collation was changed - this.SendOutputGetRequest("addproduct-params", query).Wait(); // Verify result From 1e633229d50bfd428d5c221e9357af8976baea55 Mon Sep 17 00:00:00 2001 From: luczhan Date: Thu, 13 Oct 2022 11:43:03 -0700 Subject: [PATCH 13/15] fix target path --- builds/azure-pipelines/template-steps-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 3b88ae9ea..37ed2f157 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -76,7 +76,7 @@ steps: inputs: sourceFolder: $(Build.SourcesDirectory)/src/bin/${{ parameters.configuration }}/netstandard2.0 contents: Microsoft.Azure.WebJobs.Extensions.Sql.dll - targetFolder: $(azureFunctionsExtensionBundlePath)\bin + targetFolder: $(azureFunctionsExtensionBundlePath)/bin overWrite: true - script: | From 307dfd163ef7d361696ab2eb49ff82784d006b04 Mon Sep 17 00:00:00 2001 From: luczhan Date: Thu, 13 Oct 2022 12:14:05 -0700 Subject: [PATCH 14/15] pr comments + add timeout var linux --- .../azure-pipelines/template-steps-build-test.yml | 1 + src/SqlAsyncCollector.cs | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 37ed2f157..1580c63a9 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -259,6 +259,7 @@ steps: env: SA_PASSWORD: '$(serverPassword)' AZUREFUNCTIONS_SQLBINDINGS_TELEMETRY_OPTOUT: '1' + AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MIN: '0' inputs: command: test projects: '${{ parameters.solution }}' diff --git a/src/SqlAsyncCollector.cs b/src/SqlAsyncCollector.cs index 18a6090f1..aa0c80b9b 100644 --- a/src/SqlAsyncCollector.cs +++ b/src/SqlAsyncCollector.cs @@ -59,6 +59,8 @@ internal class SqlAsyncCollector : IAsyncCollector, IDisposable private const string Collation = "Collation"; + private const int DEFAULT_TABLE_INFO_CACHE_TIMEOUT_MIN = 10; + private readonly IConfiguration _configuration; private readonly SqlAttribute _attribute; private readonly ILogger _logger; @@ -172,15 +174,15 @@ private async Task UpsertRowsAsync(IEnumerable rows, SqlAttribute attribute, ObjectCache cachedTables = MemoryCache.Default; var tableInfo = cachedTables[cacheKey] as TableInformation; - int timeout = 10; + int timeout = DEFAULT_TABLE_INFO_CACHE_TIMEOUT_MIN; string timeoutEnvVar = Environment.GetEnvironmentVariable("AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MIN"); - this._logger.LogDebugWithThreadId("TIMEOUT STRING HERE " + timeoutEnvVar); - if (!string.IsNullOrEmpty(timeoutEnvVar)) { - timeout = int.Parse(timeoutEnvVar, CultureInfo.InvariantCulture); + if (int.TryParse(timeoutEnvVar, NumberStyles.Integer, CultureInfo.InvariantCulture, out timeout)) + { + this._logger.LogDebugWithThreadId($"Overriding default table info cache timeout with new value {timeout}"); + } } - this._logger.LogDebugWithThreadId("TIMEOUT HERE " + timeout); if (tableInfo == null) { @@ -189,7 +191,7 @@ private async Task UpsertRowsAsync(IEnumerable rows, SqlAttribute attribute, tableInfo = await TableInformation.RetrieveTableInformationAsync(connection, fullTableName, this._logger, GetColumnNamesFromItem(rows.First())); var policy = new CacheItemPolicy { - // Re-look up the primary key(s) after 10 minutes (they should not change very often!) + // Re-look up the primary key(s) after timeout (default timeout is 10 minutes) AbsoluteExpiration = DateTimeOffset.Now.AddMinutes(timeout) }; From 0b7872654f8db2990d60123a3448f45411030714 Mon Sep 17 00:00:00 2001 From: luczhan Date: Thu, 13 Oct 2022 13:11:50 -0700 Subject: [PATCH 15/15] set cache to 0 for one test --- builds/azure-pipelines/template-steps-build-test.yml | 2 -- src/SqlAsyncCollector.cs | 10 +++++++--- test/Integration/IntegrationTestBase.cs | 10 +++++++++- test/Integration/SqlOutputBindingIntegrationTests.cs | 7 ++++++- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/builds/azure-pipelines/template-steps-build-test.yml b/builds/azure-pipelines/template-steps-build-test.yml index 1580c63a9..c8dd777f0 100644 --- a/builds/azure-pipelines/template-steps-build-test.yml +++ b/builds/azure-pipelines/template-steps-build-test.yml @@ -247,7 +247,6 @@ steps: TEST_SERVER: '$(testServer)' NODE_MODULES_PATH: '$(nodeModulesPath)' AZUREFUNCTIONS_SQLBINDINGS_TELEMETRY_OPTOUT: '1' - AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MIN: '0' inputs: command: test projects: '${{ parameters.solution }}' @@ -259,7 +258,6 @@ steps: env: SA_PASSWORD: '$(serverPassword)' AZUREFUNCTIONS_SQLBINDINGS_TELEMETRY_OPTOUT: '1' - AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MIN: '0' inputs: command: test projects: '${{ parameters.solution }}' diff --git a/src/SqlAsyncCollector.cs b/src/SqlAsyncCollector.cs index aa0c80b9b..07644df75 100644 --- a/src/SqlAsyncCollector.cs +++ b/src/SqlAsyncCollector.cs @@ -59,7 +59,7 @@ internal class SqlAsyncCollector : IAsyncCollector, IDisposable private const string Collation = "Collation"; - private const int DEFAULT_TABLE_INFO_CACHE_TIMEOUT_MIN = 10; + private const int AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES = 10; private readonly IConfiguration _configuration; private readonly SqlAttribute _attribute; @@ -174,14 +174,18 @@ private async Task UpsertRowsAsync(IEnumerable rows, SqlAttribute attribute, ObjectCache cachedTables = MemoryCache.Default; var tableInfo = cachedTables[cacheKey] as TableInformation; - int timeout = DEFAULT_TABLE_INFO_CACHE_TIMEOUT_MIN; - string timeoutEnvVar = Environment.GetEnvironmentVariable("AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MIN"); + int timeout = AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES; + string timeoutEnvVar = Environment.GetEnvironmentVariable("AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES"); if (!string.IsNullOrEmpty(timeoutEnvVar)) { if (int.TryParse(timeoutEnvVar, NumberStyles.Integer, CultureInfo.InvariantCulture, out timeout)) { this._logger.LogDebugWithThreadId($"Overriding default table info cache timeout with new value {timeout}"); } + else + { + timeout = AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES; + } } if (tableInfo == null) diff --git a/test/Integration/IntegrationTestBase.cs b/test/Integration/IntegrationTestBase.cs index a59f384ac..bfbe1f435 100644 --- a/test/Integration/IntegrationTestBase.cs +++ b/test/Integration/IntegrationTestBase.cs @@ -176,7 +176,7 @@ protected void StartAzurite() /// - The functionName is different than its route.
/// - You can start multiple functions by passing in a space-separated list of function names.
/// - protected void StartFunctionHost(string functionName, SupportedLanguages language, bool useTestFolder = false, DataReceivedEventHandler customOutputHandler = null) + protected void StartFunctionHost(string functionName, SupportedLanguages language, bool useTestFolder = false, DataReceivedEventHandler customOutputHandler = null, Dictionary environmentVariables = null) { string workingDirectory = language == SupportedLanguages.CSharp && useTestFolder ? GetPathToBin() : Path.Combine(GetPathToBin(), "SqlExtensionSamples", Enum.GetName(typeof(SupportedLanguages), language)); if (!Directory.Exists(workingDirectory)) @@ -196,6 +196,14 @@ protected void StartFunctionHost(string functionName, SupportedLanguages languag UseShellExecute = false }; + if (environmentVariables != null) + { + foreach (KeyValuePair variable in environmentVariables) + { + startInfo.EnvironmentVariables[variable.Key] = variable.Value; + } + } + // Always disable telemetry during test runs startInfo.EnvironmentVariables[TelemetryOptoutEnvVar] = "1"; diff --git a/test/Integration/SqlOutputBindingIntegrationTests.cs b/test/Integration/SqlOutputBindingIntegrationTests.cs index c2415849c..179810dac 100644 --- a/test/Integration/SqlOutputBindingIntegrationTests.cs +++ b/test/Integration/SqlOutputBindingIntegrationTests.cs @@ -352,7 +352,12 @@ public void AddProductWithIdentity_MissingPrimaryColumn(SupportedLanguages lang) [SqlInlineData()] public void AddProductCaseSensitiveTest(SupportedLanguages lang) { - this.StartFunctionHost(nameof(AddProductParams), lang); + // Set table info cache timeout to 0 minutes so that new collation gets picked up + var environmentVariables = new Dictionary() + { + { "AZ_FUNC_TABLE_INFO_CACHE_TIMEOUT_MINUTES", "0" } + }; + this.StartFunctionHost(nameof(AddProductParams), lang, false, null, environmentVariables); // Change database collation to case sensitive this.ExecuteNonQuery($"ALTER DATABASE {this.DatabaseName} SET Single_User WITH ROLLBACK IMMEDIATE; ALTER DATABASE {this.DatabaseName} COLLATE Latin1_General_CS_AS; ALTER DATABASE {this.DatabaseName} SET Multi_User;");