diff --git a/protobuf/README.md b/protobuf/README.md
index a307acfc..14c406e2 100644
--- a/protobuf/README.md
+++ b/protobuf/README.md
@@ -61,7 +61,7 @@ mkdir %MSGDIR%
set OUTDIR=%MSGDIR%\DotNet
mkdir %OUTDIR%
-%GRPC_TOOLS_PATH%\protoc.exe %PROTO% --csharp_out %OUTDIR% --grpc_out=%OUTDIR% --plugin=protoc-gen-grpc=%GRPC_TOOLS_PATH%\grpc_csharp_plugin.exe --proto_path=%PROTO_PATH% --proto_path=%PROTOBUF_TOOLS%
+%GRPC_TOOLS_PATH%\protoc.exe %PROTO% --csharp_out %OUTDIR% --grpc_out=%OUTDIR% --plugin=protoc-gen-grpc=%GRPC_TOOLS_PATH%\grpc_csharp_plugin.exe --proto_path=%PROTO_PATH% --proto_path=%PROTOBUF_TOOLS%
```
## JavaScript
In package.json, add to the build script the following commands to build .js files and to build .ts files. Use and install npm package `protobufjs`.
@@ -81,7 +81,10 @@ In pom.xml add following under configuration for this plugin
${basedir}//azure-functions-language-worker-protobuf/src/proto
## Python
---TODO
+```
+python -m pip install -e .[dev] -U
+python setup.py build
+```
## Contributing
diff --git a/protobuf/src/proto/FunctionRpc.proto b/protobuf/src/proto/FunctionRpc.proto
index 67a871de..49face20 100644
--- a/protobuf/src/proto/FunctionRpc.proto
+++ b/protobuf/src/proto/FunctionRpc.proto
@@ -67,10 +67,18 @@ message StreamingMessage {
// Worker logs a message back to the host
RpcLog rpc_log = 2;
-
+
FunctionEnvironmentReloadRequest function_environment_reload_request = 25;
FunctionEnvironmentReloadResponse function_environment_reload_response = 26;
+
+ // Ask the worker to close any open shared memory resources for a given invocation
+ CloseSharedMemoryResourcesRequest close_shared_memory_resources_request = 27;
+ CloseSharedMemoryResourcesResponse close_shared_memory_resources_response = 28;
+
+ // Worker indexing message types
+ FunctionsMetadataRequest functions_metadata_request = 29;
+ FunctionMetadataResponses function_metadata_responses = 30;
}
}
@@ -201,6 +209,17 @@ message FunctionEnvironmentReloadResponse {
StatusResult result = 3;
}
+// Tell the out-of-proc worker to close any shared memory maps it allocated for given invocation
+message CloseSharedMemoryResourcesRequest {
+ repeated string map_names = 1;
+}
+
+// Response from the worker indicating which of the shared memory maps have been successfully closed and which have not been closed
+// The key (string) is the map name and the value (bool) is true if it was closed, false if not
+message CloseSharedMemoryResourcesResponse {
+ map close_map_results = 1;
+}
+
// Host tells the worker to load a Function
message FunctionLoadRequest {
// unique function identifier (avoid name collisions, facilitate reload case)
@@ -245,6 +264,30 @@ message RpcFunctionMetadata {
// Is set to true for proxy
bool is_proxy = 7;
+
+ // Function indexing status
+ StatusResult status = 8;
+
+ // Function language
+ string language = 9;
+
+ // Raw binding info
+ repeated string raw_bindings = 10;
+}
+
+// Host tells worker it is ready to receive metadata
+message FunctionsMetadataRequest {
+ // base directory for function app
+ string function_app_directory = 1;
+}
+
+// Worker sends function metadata back to host
+message FunctionMetadataResponses {
+ // list of function indexing responses
+ repeated FunctionLoadRequest function_load_requests_results = 1;
+
+ // status of overall metadata request
+ StatusResult result = 2;
}
// Host requests worker to invoke a Function
@@ -263,6 +306,9 @@ message InvocationRequest {
// Populates activityId, tracestate and tags from host
RpcTraceContext trace_context = 5;
+
+ // Current retry context
+ RetryContext retry_context = 6;
}
// Host sends ActivityId, traceStateString and Tags from host
@@ -277,6 +323,18 @@ message RpcTraceContext {
map attributes = 3;
}
+// Host sends retry context for a function invocation
+message RetryContext {
+ // Current retry count
+ int32 retry_count = 1;
+
+ // Max retry count
+ int32 max_retry_count = 2;
+
+ // Exception that caused the retry
+ RpcException exception = 3;
+}
+
// Host requests worker to cancel invocation
message InvocationCancel {
// Unique id for invocation
@@ -318,6 +376,34 @@ message TypedData {
}
}
+// Specify which type of data is contained in the shared memory region being read
+enum RpcDataType {
+ unknown = 0;
+ string = 1;
+ json = 2;
+ bytes = 3;
+ stream = 4;
+ http = 5;
+ int = 6;
+ double = 7;
+ collection_bytes = 8;
+ collection_string = 9;
+ collection_double = 10;
+ collection_sint64 = 11;
+}
+
+// Used to provide metadata about shared memory region to read data from
+message RpcSharedMemory {
+ // Name of the shared memory map containing data
+ string name = 1;
+ // Offset in the shared memory map to start reading data from
+ int64 offset = 2;
+ // Number of bytes to read (starting from the offset)
+ int64 count = 3;
+ // Final type to which the read data (in bytes) is to be interpreted as
+ RpcDataType type = 4;
+}
+
// Used to encapsulate collection string
message CollectionString {
repeated string string = 1;
@@ -343,8 +429,13 @@ message ParameterBinding {
// Name for the binding
string name = 1;
- // Data for the binding
- TypedData data = 2;
+ oneof rpc_data {
+ // Data for the binding
+ TypedData data = 2;
+
+ // Metadata about the shared memory region to read data from
+ RpcSharedMemory rpc_shared_memory = 3;
+ }
}
// Used to describe a given binding on load
@@ -390,8 +481,9 @@ message RpcLog {
// Category of the log. Defaults to User if not specified.
enum RpcLogCategory {
- User = 0;
- System = 1;
+ User = 0;
+ System = 1;
+ CustomMetric = 2;
}
// Unique id for invocation (if exists)
@@ -413,11 +505,14 @@ message RpcLog {
// Exception (if exists)
RpcException exception = 6;
- // json serialized property bag, or could use a type scheme like map
+ // json serialized property bag
string properties = 7;
- // Category of the log. Either user(default) or system.
+ // Category of the log. Either user(default), system, or custom metric.
RpcLogCategory log_category = 8;
+
+ // strongly-typed (ish) property bag
+ map propertiesMap = 9;
}
// Encapsulates an Exception
@@ -484,4 +579,7 @@ message RpcHttp {
TypedData rawBody = 17;
repeated RpcClaimsIdentity identities = 18;
repeated RpcHttpCookie cookies = 19;
+ map nullable_headers = 20;
+ map nullable_params = 21;
+ map nullable_query = 22;
}
diff --git a/src/FunctionInfo.cs b/src/FunctionInfo.cs
index af8fbee8..2aea62ec 100644
--- a/src/FunctionInfo.cs
+++ b/src/FunctionInfo.cs
@@ -24,10 +24,12 @@ internal class AzFunctionInfo
{
internal const string TriggerMetadata = "TriggerMetadata";
internal const string TraceContext = "TraceContext";
+ internal const string RetryContext = "RetryContext";
internal const string DollarReturn = "$return";
internal readonly bool HasTriggerMetadataParam;
internal readonly bool HasTraceContextParam;
+ internal readonly bool HasRetryContextParam;
internal readonly string FuncDirectory;
internal readonly string FuncName;
@@ -76,6 +78,7 @@ internal AzFunctionInfo(RpcFunctionMetadata metadata)
var parametersCopy = new Dictionary(psScriptParams, StringComparer.OrdinalIgnoreCase);
HasTriggerMetadataParam = parametersCopy.Remove(TriggerMetadata);
HasTraceContextParam = parametersCopy.Remove(TraceContext);
+ HasRetryContextParam = parametersCopy.Remove(RetryContext);
var allBindings = new Dictionary(StringComparer.OrdinalIgnoreCase);
var inputBindings = new Dictionary(StringComparer.OrdinalIgnoreCase);
diff --git a/src/PowerShell/PowerShellManager.cs b/src/PowerShell/PowerShellManager.cs
index 1790a4d3..7e458914 100644
--- a/src/PowerShell/PowerShellManager.cs
+++ b/src/PowerShell/PowerShellManager.cs
@@ -199,6 +199,7 @@ public Hashtable InvokeFunction(
AzFunctionInfo functionInfo,
Hashtable triggerMetadata,
TraceContext traceContext,
+ RetryContext retryContext,
IList inputData,
FunctionInvocationPerformanceStopwatch stopwatch)
{
@@ -213,7 +214,7 @@ public Hashtable InvokeFunction(
AddEntryPointInvocationCommand(functionInfo);
stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.FunctionCodeReady);
- SetInputBindingParameterValues(functionInfo, inputData, durableController, triggerMetadata, traceContext);
+ SetInputBindingParameterValues(functionInfo, inputData, durableController, triggerMetadata, traceContext, retryContext);
stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InputBindingValuesReady);
if (!durableController.ShouldSuppressPipelineTraces())
@@ -258,7 +259,8 @@ private void SetInputBindingParameterValues(
IEnumerable inputData,
DurableController durableController,
Hashtable triggerMetadata,
- TraceContext traceContext)
+ TraceContext traceContext,
+ RetryContext retryContext)
{
foreach (var binding in inputData)
{
@@ -284,6 +286,11 @@ private void SetInputBindingParameterValues(
{
_pwsh.AddParameter(AzFunctionInfo.TraceContext, traceContext);
}
+
+ if (functionInfo.HasRetryContextParam)
+ {
+ _pwsh.AddParameter(AzFunctionInfo.RetryContext, retryContext);
+ }
}
///
diff --git a/src/Public/RetryContext.cs b/src/Public/RetryContext.cs
new file mode 100644
index 00000000..4876a4fa
--- /dev/null
+++ b/src/Public/RetryContext.cs
@@ -0,0 +1,28 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
+
+namespace Microsoft.Azure.Functions.PowerShellWorker
+{
+ ///
+ /// Custom RetryContext constructed from the RpcRetryContext member received from the host.
+ ///
+ internal class RetryContext
+ {
+ public RetryContext(int retryCount, int maxRetryCount, RpcException exception)
+ {
+ RetryCount = retryCount;
+ MaxRetryCount = maxRetryCount;
+ Exception = exception;
+ }
+
+ public int RetryCount { get; }
+
+ public int MaxRetryCount { get; }
+
+ public RpcException Exception { get; }
+ }
+}
diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs
index 0cf96b8d..6aabd3f3 100644
--- a/src/RequestProcessor.cs
+++ b/src/RequestProcessor.cs
@@ -329,9 +329,10 @@ private Hashtable InvokeFunction(
{
var triggerMetadata = GetTriggerMetadata(functionInfo, invocationRequest);
var traceContext = GetTraceContext(functionInfo, invocationRequest);
+ var retryContext = GetRetryContext(functionInfo, invocationRequest);
stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.MetadataAndTraceContextReady);
- return psManager.InvokeFunction(functionInfo, triggerMetadata, traceContext, invocationRequest.InputData, stopwatch);
+ return psManager.InvokeFunction(functionInfo, triggerMetadata, traceContext, retryContext, invocationRequest.InputData, stopwatch);
}
internal StreamingMessage ProcessInvocationCancelRequest(StreamingMessage request)
@@ -452,6 +453,19 @@ private static TraceContext GetTraceContext(AzFunctionInfo functionInfo, Invocat
invocationRequest.TraceContext.Attributes);
}
+ private static RetryContext GetRetryContext(AzFunctionInfo functionInfo, InvocationRequest invocationRequest)
+ {
+ if (!functionInfo.HasRetryContextParam)
+ {
+ return null;
+ }
+
+ return new RetryContext(
+ invocationRequest.RetryContext.RetryCount,
+ invocationRequest.RetryContext.MaxRetryCount,
+ invocationRequest.RetryContext.Exception);
+ }
+
///
/// Set the 'ReturnValue' and 'OutputData' based on the invocation results appropriately.
///
diff --git a/src/Utility/TypeExtensions.cs b/src/Utility/TypeExtensions.cs
index 112b3c52..2f3c4186 100644
--- a/src/Utility/TypeExtensions.cs
+++ b/src/Utility/TypeExtensions.cs
@@ -153,9 +153,9 @@ internal static RpcException ToRpcException(this Exception exception)
{
return new RpcException
{
- Message = exception.Message,
Source = exception.Source ?? "",
- StackTrace = exception.StackTrace ?? ""
+ StackTrace = exception.StackTrace ?? "",
+ Message = exception.Message
};
}
diff --git a/test/Unit/Function/FunctionLoaderTests.cs b/test/Unit/Function/FunctionLoaderTests.cs
index 247f8a06..dc06479e 100644
--- a/test/Unit/Function/FunctionLoaderTests.cs
+++ b/test/Unit/Function/FunctionLoaderTests.cs
@@ -100,10 +100,12 @@ public void TestFunctionLoaderGetFuncWithRequires()
Assert.Single(funcInfo.OutputBindings);
}
- [Fact]
- public void TestFunctionLoaderGetFuncWithTriggerMetadataParam()
+ [Theory]
+ [InlineData("TriggerMetadata")]
+ [InlineData("RetryContext")]
+ public void TestFunctionLoaderGetFuncWithSingleParam(string paramName)
{
- var scriptFileToUse = Path.Join(_functionDirectory, "BasicFuncScriptWithTriggerMetadata.ps1");
+ var scriptFileToUse = Path.Join(_functionDirectory, $"BasicFuncScriptWith{paramName}.ps1");
var entryPointToUse = string.Empty;
var functionLoadRequest = GetFuncLoadRequest(scriptFileToUse, entryPointToUse);
@@ -118,7 +120,7 @@ public void TestFunctionLoaderGetFuncWithTriggerMetadataParam()
Assert.Equal(3, funcInfo.FuncParameters.Count);
Assert.True(funcInfo.FuncParameters.ContainsKey("req"));
Assert.True(funcInfo.FuncParameters.ContainsKey("inputBlob"));
- Assert.True(funcInfo.FuncParameters.ContainsKey("TriggerMetadata"));
+ Assert.True(funcInfo.FuncParameters.ContainsKey(paramName));
Assert.Equal(3, funcInfo.AllBindings.Count);
Assert.Equal(2, funcInfo.InputBindings.Count);
@@ -229,10 +231,12 @@ public void ParametersShouldMatchInputBinding()
Assert.Contains("inputBlob", exception.Message);
}
- [Fact]
- public void ParametersShouldMatchInputBindingWithTriggerMetadataParam()
+ [Theory]
+ [InlineData("TriggerMetadata")]
+ [InlineData("RetryContext")]
+ public void ParametersShouldMatchInputBindingWithSingleParam(string paramName)
{
- var scriptFileToUse = Path.Join(_functionDirectory, "BasicFuncScriptWithTriggerMetadata.ps1");
+ var scriptFileToUse = Path.Join(_functionDirectory, $"BasicFuncScriptWith{paramName}.ps1");
var entryPointToUse = string.Empty;
var functionLoadRequest = GetFuncLoadRequest(scriptFileToUse, entryPointToUse);
@@ -262,9 +266,9 @@ public void EntryPointParametersShouldMatchInputBinding()
}
[Fact]
- public void EntryPointParametersShouldMatchInputBindingWithTriggerMetadataParam()
+ public void EntryPointParametersShouldMatchInputBindingWithTriggerMetadataAndRetryContextParams()
{
- var scriptFileToUse = Path.Join(_functionDirectory, "FuncWithEntryPointAndTriggerMetadata.psm1");
+ var scriptFileToUse = Path.Join(_functionDirectory, "FuncWithEntryPointAndTriggerMetadataAndRetryContext.psm1");
var entryPointToUse = "Run";
var functionLoadRequest = GetFuncLoadRequest(scriptFileToUse, entryPointToUse);
diff --git a/test/Unit/Function/TestScripts/BasicFuncScriptWithRetryContext.ps1 b/test/Unit/Function/TestScripts/BasicFuncScriptWithRetryContext.ps1
new file mode 100644
index 00000000..167e7f97
--- /dev/null
+++ b/test/Unit/Function/TestScripts/BasicFuncScriptWithRetryContext.ps1
@@ -0,0 +1,8 @@
+#
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+param($req, $inputBlob, $RetryContext)
+
+"DoNothing"
diff --git a/test/Unit/Function/TestScripts/FuncWithEntryPointAndTriggerMetadata.psm1 b/test/Unit/Function/TestScripts/FuncWithEntryPointAndTriggerMetadataAndRetryContext.psm1
similarity index 74%
rename from test/Unit/Function/TestScripts/FuncWithEntryPointAndTriggerMetadata.psm1
rename to test/Unit/Function/TestScripts/FuncWithEntryPointAndTriggerMetadataAndRetryContext.psm1
index 0ff1ab86..418f8d99 100644
--- a/test/Unit/Function/TestScripts/FuncWithEntryPointAndTriggerMetadata.psm1
+++ b/test/Unit/Function/TestScripts/FuncWithEntryPointAndTriggerMetadataAndRetryContext.psm1
@@ -5,7 +5,7 @@
function Run
{
- param($req, $inputBlob, $TriggerMetadata)
+ param($req, $inputBlob, $TriggerMetadata, $RetryContext)
"Run"
}
diff --git a/test/Unit/PowerShell/PowerShellManagerTests.cs b/test/Unit/PowerShell/PowerShellManagerTests.cs
index 7fb062aa..5d4faf42 100644
--- a/test/Unit/PowerShell/PowerShellManagerTests.cs
+++ b/test/Unit/PowerShell/PowerShellManagerTests.cs
@@ -38,6 +38,15 @@ public class PowerShellManagerTests : IDisposable
private const string TestInputBindingName = "req";
private const string TestOutputBindingName = "res";
private const string TestStringData = "Foo";
+ private const int TestRetryCount = 0;
+ private const int TestMaxRetryCount = 1;
+ private const string TestMessage = "TestMessage";
+ private readonly static RpcException TestException = new RpcException
+ {
+ Source = "",
+ StackTrace = "",
+ Message = TestMessage
+ };
private readonly static string s_funcDirectory;
private readonly static FunctionLoadRequest s_functionLoadRequest;
@@ -176,9 +185,9 @@ public void InvokeBasicFunctionWithRequiresWorks()
}
[Fact]
- public void InvokeBasicFunctionWithTriggerMetadataAndTraceContextWorks()
+ public void InvokeBasicFunctionWithTriggerMetadataAndTraceContextAndRetryContextWorks()
{
- string path = Path.Join(s_funcDirectory, "testBasicFunctionWithTriggerMetadata.ps1");
+ string path = Path.Join(s_funcDirectory, "testBasicFunctionWithTriggerMetadataAndRetryContext.ps1");
var (functionInfo, testManager) = PrepareFunction(path, string.Empty);
Hashtable triggerMetadata = new Hashtable(StringComparer.OrdinalIgnoreCase)
@@ -186,18 +195,20 @@ public void InvokeBasicFunctionWithTriggerMetadataAndTraceContextWorks()
{ TestInputBindingName, TestStringData }
};
+ RetryContext retryContext = new RetryContext(TestRetryCount, TestMaxRetryCount, TestException);
+
try
{
FunctionMetadata.RegisterFunctionMetadata(testManager.InstanceId, functionInfo.OutputBindings);
- Hashtable result = InvokeFunction(testManager, functionInfo, triggerMetadata);
+ Hashtable result = InvokeFunction(testManager, functionInfo, triggerMetadata, retryContext);
// The outputBinding hashtable for the runspace should be cleared after 'InvokeFunction'
Hashtable outputBindings = FunctionMetadata.GetOutputBindingHashtable(testManager.InstanceId);
Assert.Empty(outputBindings);
// A PowerShell function should be created fro the Az function.
- string expectedResult = $"{TestStringData},{functionInfo.DeployedPSFuncName}";
+ string expectedResult = $"{TestStringData},{functionInfo.DeployedPSFuncName}:{TestRetryCount},{TestMaxRetryCount},{TestMessage}";
Assert.Equal(expectedResult, result[TestOutputBindingName]);
}
finally
@@ -391,7 +402,7 @@ internal void SuppressPipelineTracesForDurableActivityFunctionOnly(DurableFuncti
{
FunctionMetadata.RegisterFunctionMetadata(testManager.InstanceId, functionInfo.OutputBindings);
- var result = testManager.InvokeFunction(functionInfo, null, null, CreateOrchestratorInputData(), new FunctionInvocationPerformanceStopwatch());
+ var result = testManager.InvokeFunction(functionInfo, null, null, null, CreateOrchestratorInputData(), new FunctionInvocationPerformanceStopwatch());
var relevantLogs = s_testLogger.FullLog.Where(message => message.StartsWith("Information: OUTPUT:")).ToList();
var expected = shouldSuppressPipelineTraces ? new string[0] : new[] { "Information: OUTPUT: Hello" };
@@ -421,9 +432,13 @@ private static List CreateOrchestratorInputData()
return testInputData;
}
- private static Hashtable InvokeFunction(PowerShellManager powerShellManager, AzFunctionInfo functionInfo, Hashtable triggerMetadata = null)
+ private static Hashtable InvokeFunction(
+ PowerShellManager powerShellManager,
+ AzFunctionInfo functionInfo,
+ Hashtable triggerMetadata = null,
+ RetryContext retryContext = null)
{
- return powerShellManager.InvokeFunction(functionInfo, triggerMetadata, null, s_testInputData, new FunctionInvocationPerformanceStopwatch());
+ return powerShellManager.InvokeFunction(functionInfo, triggerMetadata, null, retryContext, s_testInputData, new FunctionInvocationPerformanceStopwatch());
}
private class ContextValidatingLogger : ILogger
diff --git a/test/Unit/PowerShell/TestScripts/testBasicFunctionWithTriggerMetadata.ps1 b/test/Unit/PowerShell/TestScripts/testBasicFunctionWithTriggerMetadataAndRetryContext.ps1
similarity index 56%
rename from test/Unit/PowerShell/TestScripts/testBasicFunctionWithTriggerMetadata.ps1
rename to test/Unit/PowerShell/TestScripts/testBasicFunctionWithTriggerMetadataAndRetryContext.ps1
index 401399c4..4c10f0ad 100644
--- a/test/Unit/PowerShell/TestScripts/testBasicFunctionWithTriggerMetadata.ps1
+++ b/test/Unit/PowerShell/TestScripts/testBasicFunctionWithTriggerMetadataAndRetryContext.ps1
@@ -3,11 +3,16 @@
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
-param ($Req, $TriggerMetadata)
+param ($Req, $TriggerMetadata, $RetryContext)
# Used for logging tests
Write-Verbose "a log"
$cmdName = $MyInvocation.MyCommand.Name
-$result = "{0},{1}" -f $TriggerMetadata.Req, $cmdName
+$result = "{0},{1}:{2},{3},{4}" -f `
+ $TriggerMetadata.Req,`
+ $cmdName,`
+ $RetryContext.RetryCount,`
+ $RetryContext.MaxRetryCount,`
+ $RetryContext.Exception.Message
Push-OutputBinding -Name res -Value $result