Skip to content

Commit

Permalink
[v3x] Include minimum durable payload to sync trigger (#8292)
Browse files Browse the repository at this point in the history 
* Include minimum durable payload to synctrigger

* update release note

* added missed change on v4
  • Loading branch information
@TsuyoshiUshio
TsuyoshiUshio committed Apr 8, 2022
1 parent eb93c2a commit 2d6b315
Show file tree
Hide file tree
Showing 8 changed files with 292 additions and 14 deletions.
1 change: 1 addition & 0 deletions release_notes.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
-->
- Read FuncitonsHostingConfigurations to enable worker concurrency feature (#8125)
- Updated Java Worker Version to [1.11.0](https://github.com/Azure/azure-functions-java-worker/releases/tag/1.11.0)
- Add Include minimum durable payload to sync trigger (#8292)

**Release sprint:** Sprint 118
[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Afeature+is%3Aclosed) ]
37 changes: 36 additions & 1 deletion src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public class FunctionsSyncManager : IFunctionsSyncManager, IDisposable
private const string DurableTaskV1StorageConnectionName = "azureStorageConnectionStringName";
private const string DurableTaskV2StorageOptions = "storageProvider";
private const string DurableTaskV2StorageConnectionName = "connectionStringName";
private const string DurableTaskV2MaxConcurrentActivityFunctions = "maxConcurrentActivityFunctions";
private const string DurableTaskV2MaxConcurrentOrchestratorFunctions = "maxConcurrentOrchestratorFunctions";
private const string DurableTask = "durableTask";

// 45 alphanumeric characters gives us a buffer in our table/queue/blob container names.
Expand Down Expand Up @@ -483,6 +485,21 @@ private static JObject UpdateDurableFunctionConfig(JObject trigger, DurableConfi
{
trigger[Connection] = durableTaskConfig.Connection;
}

if (durableTaskConfig.StorageProvider != null)
{
trigger[DurableTaskV2StorageOptions] = durableTaskConfig.StorageProvider;
}

if (durableTaskConfig.MaxConcurrentOrchestratorFunctions != 0)
{
trigger[DurableTaskV2MaxConcurrentOrchestratorFunctions] = durableTaskConfig.MaxConcurrentOrchestratorFunctions;
}

if (durableTaskConfig.MaxConcurrentActivityFunctions != 0)
{
trigger[DurableTaskV2MaxConcurrentActivityFunctions] = durableTaskConfig.MaxConcurrentActivityFunctions;
}
}
return trigger;
}
Expand Down Expand Up @@ -597,6 +614,18 @@ private DurableConfig GetDurableV2Config(JObject durableHostConfig)
{
config.Connection = nameValue.ToString();
}

config.StorageProvider = storageOptions;
}

if (durableHostConfig.TryGetValue(DurableTaskV2MaxConcurrentOrchestratorFunctions, StringComparison.OrdinalIgnoreCase, out JToken maxConcurrentOrchestratorFunctions) && maxConcurrentOrchestratorFunctions != null)
{
config.MaxConcurrentOrchestratorFunctions = int.Parse(maxConcurrentOrchestratorFunctions.ToString());
}

if (durableHostConfig.TryGetValue(DurableTaskV2MaxConcurrentActivityFunctions, StringComparison.OrdinalIgnoreCase, out JToken maxConcurrentActivityFunctions) && maxConcurrentActivityFunctions != null)
{
config.MaxConcurrentActivityFunctions = int.Parse(maxConcurrentActivityFunctions.ToString());
}
}

Expand Down Expand Up @@ -723,9 +752,15 @@ private class DurableConfig

public string Connection { get; set; }

public JToken StorageProvider { get; set; }

public int MaxConcurrentActivityFunctions { get; set; }

public int MaxConcurrentOrchestratorFunctions { get; set; }

public bool HasValues()
{
return this.HubName != null || this.Connection != null;
return this.HubName != null || this.Connection != null || this.StorageProvider != null || this.MaxConcurrentOrchestratorFunctions != 0 || this.MaxConcurrentOrchestratorFunctions != 0;
}
}
}
Expand Down
113 changes: 100 additions & 13 deletions test/WebJobs.Script.Tests.Integration/Management/FunctionsSyncManagerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,13 @@ public FunctionsSyncManagerTests()
_functionsSyncManager = new FunctionsSyncManager(configuration, hostIdProviderMock.Object, optionsMonitor, loggerFactory.CreateLogger<FunctionsSyncManager>(), httpClient, _secretManagerProviderMock.Object, _mockWebHostEnvironment.Object, _mockEnvironment.Object, _hostNameProvider, functionMetadataManager, azureStorageProvider);
}

private string GetExpectedSyncTriggersPayload(string postedConnection = DefaultTestConnection, string postedTaskHub = DefaultTestTaskHub)
private string GetExpectedSyncTriggersPayload(string postedConnection = DefaultTestConnection, string postedTaskHub = DefaultTestTaskHub, string durableVersion = "V2")
{
string taskHubSegment = postedTaskHub != null ? $",\"taskHubName\":\"{postedTaskHub}\"" : "";
string storageProviderSegment = postedConnection != null && durableVersion == "V2" ? $",\"storageProvider\":{{\"connectionStringName\":\"DurableConnection\"}}" : "";
return "[{\"authLevel\":\"anonymous\",\"type\":\"httpTrigger\",\"direction\":\"in\",\"name\":\"req\",\"functionName\":\"function1\"}," +
$"{{\"name\":\"myQueueItem\",\"type\":\"orchestrationTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function2\"{taskHubSegment}}}," +
$"{{\"name\":\"myQueueItem\",\"type\":\"activityTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function3\"{taskHubSegment}}}]";
$"{{\"name\":\"myQueueItem\",\"type\":\"orchestrationTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function2\"{taskHubSegment}{storageProviderSegment}}}," +
$"{{\"name\":\"myQueueItem\",\"type\":\"activityTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function3\"{taskHubSegment}{storageProviderSegment}}}]";
}

private void ResetMockFileSystem(string hostJsonContent = null, string extensionsJsonContent = null)
Expand Down Expand Up @@ -235,18 +236,18 @@ public async Task TrySyncTriggers_PostsExpectedContent(bool cacheEnabled)

if (cacheEnabled)
{
VerifyResultWithCacheOn();
VerifyResultWithCacheOn(durableVersion: "V1");
}
else
{
VerifyResultWithCacheOff();
VerifyResultWithCacheOff(durableVersion: "V1");
}
}
}

private void VerifyResultWithCacheOn(string connection = DefaultTestConnection, string expectedTaskHub = "TestHubValue")
private void VerifyResultWithCacheOn(string connection = DefaultTestConnection, string expectedTaskHub = "TestHubValue", string durableVersion = "V2")
{
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(postedConnection: connection, postedTaskHub: expectedTaskHub);
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(postedConnection: connection, postedTaskHub: expectedTaskHub, durableVersion);
// verify triggers
var result = JObject.Parse(_contentBuilder.ToString());
var triggers = result["triggers"];
Expand Down Expand Up @@ -286,9 +287,9 @@ private void VerifyResultWithCacheOn(string connection = DefaultTestConnection,
Assert.False(triggersLog.Contains("secrets"));
}

private void VerifyResultWithCacheOff()
private void VerifyResultWithCacheOff(string durableVersion)
{
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload();
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(durableVersion: durableVersion);
var triggers = JArray.Parse(_contentBuilder.ToString());
Assert.Equal(expectedSyncTriggersPayload, triggers.ToString(Formatting.None));

Expand Down Expand Up @@ -363,7 +364,7 @@ public async Task TrySyncTriggers_BackgroundSync_PostsExpectedContent()
Assert.Equal(1, _mockHttpHandler.RequestCount);
var result = JObject.Parse(_contentBuilder.ToString());
var triggers = result["triggers"];
Assert.Equal(GetExpectedSyncTriggersPayload(), triggers.ToString(Formatting.None));
Assert.Equal(GetExpectedSyncTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));

string hash = string.Empty;
var downloadResponse = await hashBlob.DownloadAsync();
Expand Down Expand Up @@ -429,7 +430,7 @@ public async Task TrySyncTriggers_BackgroundSync_SetTriggersFailure_HashNotUpdat
Assert.Equal(1, _mockHttpHandler.RequestCount);
var result = JObject.Parse(_contentBuilder.ToString());
var triggers = result["triggers"];
Assert.Equal(GetExpectedSyncTriggersPayload(), triggers.ToString(Formatting.None));
Assert.Equal(GetExpectedSyncTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
bool hashBlobExists = await hashBlob.ExistsAsync();
Assert.False(hashBlobExists);

Expand Down Expand Up @@ -488,7 +489,7 @@ public async Task TrySyncTriggers_NoDurableTaskHub_DurableV1ExtensionJson_V1Defa
Assert.True(syncResult.Success, "SyncTriggers should return success true");
Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");

VerifyResultWithCacheOn(expectedTaskHub: null, connection: null);
VerifyResultWithCacheOn(expectedTaskHub: null, connection: null, durableVersion: "V1");
}
}

Expand Down Expand Up @@ -567,7 +568,7 @@ public async Task TrySyncTriggers_DurableV1ExtensionJson_V1ConfigPosted()
Assert.True(syncResult.Success, "SyncTriggers should return success true");
Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");

VerifyResultWithCacheOn(expectedTaskHub: "DurableTask", connection: "DurableConnection");
VerifyResultWithCacheOn(expectedTaskHub: "DurableTask", connection: "DurableConnection", durableVersion: "V1");
}
}

Expand Down Expand Up @@ -602,6 +603,92 @@ public async Task TrySyncTriggers_DurableV2ExtensionJson_V2ConfigPosted()
}
}

[Theory]
[InlineData("DurableMsSQLProviderPayload", "DurableMsSQLProviderPayload")]
[InlineData("DurableAdditionalPayload", "DurableMsSQLProviderPayload")] // Payload trimed to the minimum payload
[InlineData("DurableHasHubNameAndOrchestrationConfig", "DurableHasHubNameAndOrchestrationConfigPayload")]
[InlineData("DurableHasStorageAndActivityConfig", "DurableHasStorageAndActivityConfigPayload")]
[InlineData("Empty", "EmptyDurablePayload")]
public async Task TrySyncTriggers_DurableV2ExtensionJson_StorageProviderPosted(string scenarioName, string expectedPayload)
{
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteName)).Returns("TestHubValue");
using (var env = new TestScopedEnvironmentVariable(_vars))
{
var durableConfig = GetDurableConfig(scenarioName);

var hostConfig = GetHostConfig(durableConfig, useBundles: false);

// See what happens when extension.json is not present but bundles are used.
SetupDurableExtension(hostConfig.ToString(), durableExtensionJsonVersion: "2.0.0.0");

// Act
var syncResult = await _functionsSyncManager.TrySyncTriggersAsync();

// Assert
Assert.True(syncResult.Success, "SyncTriggers should return success true");
Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");

//Verify Result
var result = JObject.Parse(_contentBuilder.ToString());
var triggers = result["triggers"];
var triggersString = triggers.ToString();
Assert.Equal(GetPayloadFromFile($"{expectedPayload}.json"), triggers.ToString());
}
}

private JObject GetDurableConfig(string scenarioName)
{
var durableConfig = new JObject();
durableConfig["hubName"] = "DurableTask";

var azureStorageConfig = new JObject();
azureStorageConfig["type"] = "mssql";
azureStorageConfig["connectionStringName"] = "SQLDB_Connection";
azureStorageConfig["taskEventLockTimeout"] = "00:02:00";
azureStorageConfig["createDatabaseIfNotExists"] = true;
durableConfig["storageProvider"] = azureStorageConfig;
durableConfig["maxConcurrentActivityFunctions"] = 12;
durableConfig["maxConcurrentOrchestratorFunctions"] = 10;

switch (scenarioName)
{
case "DurableMsSQLProviderPayload":
return durableConfig;
case "DurableAdditionalPayload":
// These additional parameters are exptected to be ignored.
durableConfig["extendedSessionsEnabled"] = true;
durableConfig["extendedSessionIdleTimeoutInSeconds"] = 30;
return durableConfig;
case "DurableHasHubNameAndOrchestrationConfig":
durableConfig = new JObject();
durableConfig["hubName"] = "DurableTask";
durableConfig["maxConcurrentOrchestratorFunctions"] = 10;
return durableConfig;
case "DurableHasStorageAndActivityConfig":
durableConfig = new JObject();
azureStorageConfig = new JObject();
azureStorageConfig["type"] = "mssql";
azureStorageConfig["connectionStringName"] = "SQLDB_Connection";
azureStorageConfig["taskEventLockTimeout"] = "00:02:00";
azureStorageConfig["createDatabaseIfNotExists"] = true;
durableConfig["storageProvider"] = azureStorageConfig;
durableConfig["maxConcurrentActivityFunctions"] = 12;
return durableConfig;
case "Empty":
return new JObject();
default: return new JObject();
}

}


private string GetPayloadFromFile(string fileName)
{
var fullPath = Path.Combine("Management", "Payload", fileName);
return File.ReadAllText(fullPath);
}


[Fact]
public async Task UpdateHashAsync_Succeeds()
{
Expand Down
29 changes: 29 additions & 0 deletions ....Tests.Integration/Management/Payload/DurableHasHubNameAndOrchestrationConfigPayload.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
[
{
"authLevel": "anonymous",
"type": "httpTrigger",
"direction": "in",
"name": "req",
"functionName": "function1"
},
{
"name": "myQueueItem",
"type": "orchestrationTrigger",
"direction": "in",
"queueName": "myqueue-items",
"connection": "",
"functionName": "function2",
"taskHubName": "DurableTask",
"maxConcurrentOrchestratorFunctions": 10
},
{
"name": "myQueueItem",
"type": "activityTrigger",
"direction": "in",
"queueName": "myqueue-items",
"connection": "",
"functionName": "function3",
"taskHubName": "DurableTask",
"maxConcurrentOrchestratorFunctions": 10
}
]
41 changes: 41 additions & 0 deletions ...cript.Tests.Integration/Management/Payload/DurableHasStorageAndActivityConfigPayload.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
[
{
"authLevel": "anonymous",
"type": "httpTrigger",
"direction": "in",
"name": "req",
"functionName": "function1"
},
{
"name": "myQueueItem",
"type": "orchestrationTrigger",
"direction": "in",
"queueName": "myqueue-items",
"connection": "SQLDB_Connection",
"functionName": "function2",
"taskHubName": "TestHubValue",
"storageProvider": {
"type": "mssql",
"connectionStringName": "SQLDB_Connection",
"taskEventLockTimeout": "00:02:00",
"createDatabaseIfNotExists": true
},
"maxConcurrentActivityFunctions": 12
},
{
"name": "myQueueItem",
"type": "activityTrigger",
"direction": "in",
"queueName": "myqueue-items",
"connection": "SQLDB_Connection",
"functionName": "function3",
"taskHubName": "TestHubValue",
"storageProvider": {
"type": "mssql",
"connectionStringName": "SQLDB_Connection",
"taskEventLockTimeout": "00:02:00",
"createDatabaseIfNotExists": true
},
"maxConcurrentActivityFunctions": 12
}
]
43 changes: 43 additions & 0 deletions test/WebJobs.Script.Tests.Integration/Management/Payload/DurableMsSQLProviderPayload.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
[
{
"authLevel": "anonymous",
"type": "httpTrigger",
"direction": "in",
"name": "req",
"functionName": "function1"
},
{
"name": "myQueueItem",
"type": "orchestrationTrigger",
"direction": "in",
"queueName": "myqueue-items",
"connection": "SQLDB_Connection",
"functionName": "function2",
"taskHubName": "DurableTask",
"storageProvider": {
"type": "mssql",
"connectionStringName": "SQLDB_Connection",
"taskEventLockTimeout": "00:02:00",
"createDatabaseIfNotExists": true
},
"maxConcurrentOrchestratorFunctions": 10,
"maxConcurrentActivityFunctions": 12
},
{
"name": "myQueueItem",
"type": "activityTrigger",
"direction": "in",
"queueName": "myqueue-items",
"connection": "SQLDB_Connection",
"functionName": "function3",
"taskHubName": "DurableTask",
"storageProvider": {
"type": "mssql",
"connectionStringName": "SQLDB_Connection",
"taskEventLockTimeout": "00:02:00",
"createDatabaseIfNotExists": true
},
"maxConcurrentOrchestratorFunctions": 10,
"maxConcurrentActivityFunctions": 12
}
]

0 comments on commit 2d6b315

Please sign in to comment.