Durable entity invocation from orchestrator seems to have been broken with Microsoft.DurableTask.Client/Worker v1.6.0 #385
Description
I just upgraded durable functions packages Microsoft.DurableTask.Client/Worker from v1.4.0 to v1.6.0 (latest) and performing some tests of my .NET 8 isolated functions application, I ran into an issue when trying to call a durable entity from an orchestrator function.
More specifically, upon successful return from the invoked entity's operation, I get the following ArgumentNullException at the orchestrator. Note that I'm having the exact same behavior both when locally debugging and at the Azure deployed application.
As I understand from the stack trace, the problem seems to be a null value assignment to the EntityBatchResult's CompletionToken property (bold line below).
[2025-02-27T15:15:22.099Z] Function 'ContractDailyTranEntity', Invocation id 'b3b58b5a-ab9e-4a04-8765-427b8d38bc12': An exception was thrown by the invocation.
[2025-02-27T15:15:22.101Z] Result: Function 'ContractDailyTranEntity', Invocation id 'b3b58b5a-ab9e-4a04-8765-427b8d38bc12': An exception was thrown by the invocation.
Exception: System.ArgumentNullException: Value cannot be null. (Parameter 'value')
[2025-02-27T15:15:22.102Z] at Google.Protobuf.ProtoPreconditions.CheckNotNull[T](T value, String name)
[2025-02-27T15:15:22.103Z] at Microsoft.DurableTask.Protobuf.EntityBatchResult.set_CompletionToken(String value)
[2025-02-27T15:15:22.103Z] at Microsoft.DurableTask.ProtoUtils.ToEntityBatchResult(EntityBatchResult entityBatchResult, String completionToken, IEnumerable'1 operationInfos)
[2025-02-27T15:15:22.104Z] at Microsoft.DurableTask.Worker.Grpc.GrpcEntityRunner.LoadAndRunAsync(String encodedEntityRequest, ITaskEntity implementation, IServiceProvider services)
[2025-02-27T15:15:22.105Z] at Microsoft.Azure.Functions.Worker.TaskEntityDispatcher.DispatchAsync(ITaskEntity entity) in //src/Worker.Extensions.DurableTask/TaskEntityDispatcher.cs:line 43
[2025-02-27T15:15:22.106Z] at BSS.AzureFunctions.DirectFunctionExecutor.ExecuteAsync(FunctionContext context) in D:\WORK\tfsrl\BSS\Code\Dev\BSS.AzureFunctions\obj\Debug\net8.0\Microsoft.Azure.Functions.Worker.Sdk.Generators\Microsoft.Azure.Functions.Worker.Sdk.Generators.FunctionExecutorGenerator\GeneratedFunctionExecutor.g.cs:line 46
[2025-02-27T15:15:22.107Z] at Microsoft.Azure.Functions.Worker.OutputBindings.OutputBindingsMiddleware.Invoke(FunctionContext context, FunctionExecutionDelegate next) in D:\a_work\1\s\src\DotNetWorker.Core\OutputBindings\OutputBindingsMiddleware.cs:line 13
[2025-02-27T15:15:22.108Z] at Microsoft.Azure.Functions.Worker.Extensions.DurableTask.DurableTaskFunctionsMiddleware.RunEntityAsync(FunctionContext context, BindingMetadata triggerBinding, FunctionExecutionDelegate next) in //src/Worker.Extensions.DurableTask/DurableTaskFunctionsMiddleware.cs:line 94
[2025-02-27T15:15:22.109Z] at Microsoft.Azure.Functions.Worker.FunctionsApplication.InvokeFunctionAsync(FunctionContext context) in D:\a_work\1\s\src\DotNetWorker.Core\FunctionsApplication.cs:line 96
Stack: at Google.Protobuf.ProtoPreconditions.CheckNotNull[T](T value, String name)
[2025-02-27T15:15:22.110Z] at Microsoft.DurableTask.Protobuf.EntityBatchResult.set_CompletionToken(String value)
[2025-02-27T15:15:22.112Z] at Microsoft.DurableTask.ProtoUtils.ToEntityBatchResult(EntityBatchResult entityBatchResult, String completionToken, IEnumerable'1 operationInfos)
[2025-02-27T15:15:22.112Z] at Microsoft.DurableTask.Worker.Grpc.GrpcEntityRunner.LoadAndRunAsync(String encodedEntityRequest, ITaskEntity implementation, IServiceProvider services)
[2025-02-27T15:15:22.113Z] at Microsoft.Azure.Functions.Worker.TaskEntityDispatcher.DispatchAsync(ITaskEntity entity) in //src/Worker.Extensions.DurableTask/TaskEntityDispatcher.cs:line 43
[2025-02-27T15:15:22.114Z] at BSS.AzureFunctions.DirectFunctionExecutor.ExecuteAsync(FunctionContext context) in D:\WORK\tfsrl\BSS\Code\Dev\BSS.AzureFunctions\obj\Debug\net8.0\Microsoft.Azure.Functions.Worker.Sdk.Generators\Microsoft.Azure.Functions.Worker.Sdk.Generators.FunctionExecutorGenerator\GeneratedFunctionExecutor.g.cs:line 46
[2025-02-27T15:15:22.115Z] at Microsoft.Azure.Functions.Worker.OutputBindings.OutputBindingsMiddleware.Invoke(FunctionContext context, FunctionExecutionDelegate next) in D:\a_work\1\s\src\DotNetWorker.Core\OutputBindings\OutputBindingsMiddleware.cs:line 13
[2025-02-27T15:15:22.116Z] at Microsoft.Azure.Functions.Worker.Extensions.DurableTask.DurableTaskFunctionsMiddleware.RunEntityAsync(FunctionContext context, BindingMetadata triggerBinding, FunctionExecutionDelegate next) in //src/Worker.Extensions.DurableTask/DurableTaskFunctionsMiddleware.cs:line 94
[2025-02-27T15:15:22.117Z] at Microsoft.Azure.Functions.Worker.FunctionsApplication.InvokeFunctionAsync(FunctionContext context) in D:\a_work\1\s\src\DotNetWorker.Core\FunctionsApplication.cs:line 96.
[2025-02-27T15:15:22.332Z] Executed 'Functions.ContractDailyTranEntity' (Failed, Id=b3b58b5a-ab9e-4a04-8765-427b8d38bc12, Duration=38719ms)
[2025-02-27T15:15:22.334Z] System.Private.CoreLib: Exception while executing function: Functions.ContractDailyTranEntity. System.Private.CoreLib: Result: Failure
Exception: Value cannot be null. (Parameter 'value')
Stack: at Google.Protobuf.ProtoPreconditions.CheckNotNull[T](T value, String name)
[2025-02-27T15:15:22.335Z] at Microsoft.DurableTask.Protobuf.EntityBatchResult.set_CompletionToken(String value)
[2025-02-27T15:15:22.336Z] at Microsoft.DurableTask.ProtoUtils.ToEntityBatchResult(EntityBatchResult entityBatchResult, String completionToken, IEnumerable`1 operationInfos)
[2025-02-27T15:15:22.336Z] at Microsoft.DurableTask.Worker.Grpc.GrpcEntityRunner.LoadAndRunAsync(String encodedEntityRequest, ITaskEntity implementation, IServiceProvider services)
[2025-02-27T15:15:22.337Z] at Microsoft.Azure.Functions.Worker.TaskEntityDispatcher.DispatchAsync(ITaskEntity entity) in //src/Worker.Extensions.DurableTask/TaskEntityDispatcher.cs:line 43
[2025-02-27T15:15:22.338Z] at BSS.AzureFunctions.DirectFunctionExecutor.ExecuteAsync(FunctionContext context) in D:\WORK\tfsrl\BSS\Code\Dev\BSS.AzureFunctions\obj\Debug\net8.0\Microsoft.Azure.Functions.Worker.Sdk.Generators\Microsoft.Azure.Functions.Worker.Sdk.Generators.FunctionExecutorGenerator\GeneratedFunctionExecutor.g.cs:line 46
[2025-02-27T15:15:22.339Z] at Microsoft.Azure.Functions.Worker.OutputBindings.OutputBindingsMiddleware.Invoke(FunctionContext context, FunctionExecutionDelegate next) in D:\a_work\1\s\src\DotNetWorker.Core\OutputBindings\OutputBindingsMiddleware.cs:line 13
[2025-02-27T15:15:22.339Z] at Microsoft.Azure.Functions.Worker.Extensions.DurableTask.DurableTaskFunctionsMiddleware.RunEntityAsync(FunctionContext context, BindingMetadata triggerBinding, FunctionExecutionDelegate next) in //src/Worker.Extensions.DurableTask/DurableTaskFunctionsMiddleware.cs:line 94
[2025-02-27T15:15:22.340Z] at Microsoft.Azure.Functions.Worker.FunctionsApplication.InvokeFunctionAsync(FunctionContext context) in D:\a_work\1\s\src\DotNetWorker.Core\FunctionsApplication.cs:line 96
[2025-02-27T15:15:22.341Z] at Microsoft.Azure.Functions.Worker.Handlers.InvocationHandler.InvokeAsync(InvocationRequest request) in D:\a_work\1\s\src\DotNetWorker.Grpc\Handlers\InvocationHandler.cs:line 89.
[2025-02-27T15:15:22.345Z] @contractdailytranentity@20250224_fc3299ab-3df1-480d-9464-bfada1618917: Function 'contractdailytranentity (Entity)' failed with an error. Reason: Microsoft.Azure.WebJobs.Host.FunctionInvocationException
at Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor.ExecuteWithLoggingAsync(IFunctionInstanceEx instance, FunctionStartedMessage message, FunctionInstanceLogEntry instanceLogEntry, ParameterHelper parameterHelper, ILogger logger, CancellationToken cancellationToken) in D:\a_work\1\s\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs:line 352
[2025-02-27T15:15:22.347Z] at Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor.TryExecuteAsync(IFunctionInstance functionInstance, CancellationToken cancellationToken) in D:\a_work\1\s\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs:line 108. IsReplay: False. State: Failed. RuntimeStatus: Failed. HubName: TestHubName. AppName: . SlotName: . ExtensionVersion: 3.0.2. SequenceNumber: 7. TaskEventId: -1
I rolled back to durable functions packages v1.4.0 and the application works fine. I even tested with the 1.5.0 packages version and the application still works fine, so I guess that the issue is first introduced with the v1.6.0 packages.
Please find below the project's package references, as well as fragments of the involved code.
<PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.CosmosDB" Version="4.12.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" Version="1.2.2" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.3.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Storage" Version="6.6.1" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues" Version="5.5.1" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Timer" Version="4.3.1" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.1" />
<PackageReference Include="Microsoft.DurableTask.Client" Version="1.6.0" />
<PackageReference Include="Microsoft.DurableTask.Client.Grpc" Version="1.6.0" />
<PackageReference Include="Microsoft.DurableTask.Generators" Version="1.0.0-preview.1" />
<PackageReference Include="Microsoft.DurableTask.Worker" Version="1.6.0" />
<PackageReference Include="Microsoft.DurableTask.Worker.Grpc" Version="1.6.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.2" />
[Function(nameof(CloseDayOrchestrator))]
public static async Task<string?> CloseDayOrchestrator([OrchestrationTrigger] TaskOrchestrationContext context, CloseDayOrchestratorParams input, FunctionContext functionContext)
{
ILogger logger = context.CreateReplaySafeLogger(nameof(CloseDayOrchestrator));
try
{
ContractDailyTranEntity.ApplyData applyData = new() { Id = "1", Date = new DateTime(2025, 2, 24), Items = new List<int>() { 1, 2, 3 }};
EntityInstanceId entityInstanceId = new EntityInstanceId(nameof(ContractDailyTranEntity), "20250224_fc3299ab-3df1-480d-9464-bfada1618917");
await context.Entities.CallEntityAsync(entityInstanceId, nameof(ContractDailyTranEntity.Apply), input: applyData);
}
catch (Exception ex)
{
logger.LogError(ex, "Execution failed");
}
return null;
}
public class ContractDailyTranEntity : TaskEntity<ContractDailyTranEntity.StateData?>
{
public class StateData
{
public string Id { get; set; }
public DateTime Date { get; set; }
}
public class ApplyData
{
public string Id { get; set; }
public DateTime Date { get; set; }
public List<int> Items { get; set; }
}
private readonly ILogger _Logger;
public ContractDailyTranEntity(ILogger<ContractDailyTranEntity> logger)
{
_Logger = logger;
}
protected override StateData? InitializeState(TaskEntityOperation entityOperation)
{
if (entityOperation.Name == nameof(Apply))
{
ApplyData? input = entityOperation.GetInput<ApplyData>();
if (input == null)
throw new Exception("ApplyData not provided");
//some other work here
StateData state = new() { Id = input.Id, Date = input.Date };
return state;
}
else
throw new Exception("State initialization not expected as of the current operation");
}
public async Task Apply(ApplyData input)
{
try
{
//some work here
this.State = null; //entity state not necessary anymore, delete it <== note that this doesn't affect the erroneous behavior, exception is still thrown to the orchestrator if state is not deleted
}
catch (Exception ex)
{
_Logger.LogError(ex, "Error applying data");
}
}
[Function(nameof(ContractDailyTranEntity))]
public static Task Run([EntityTrigger] TaskEntityDispatcher dispatcher)
{
return dispatcher.DispatchAsync<ContractDailyTranEntity>();
}
}