From 07a4d601eb832efd227acd6a9863915a85d0c350 Mon Sep 17 00:00:00 2001 From: Kannan Rajah Date: Tue, 3 Feb 2026 23:07:27 -0800 Subject: [PATCH 1/3] Store worker_instance_key in ActivityInfo --- api/persistence/v1/executions.pb.go | 18 +++++++++++---- go.mod | 1 + go.sum | 2 -- .../api/persistence/v1/executions.proto | 3 +++ .../api/recordactivitytaskstarted/api.go | 1 + .../api/respondactivitytaskcompleted/api.go | 23 ++++++++++--------- .../workflow_task_completed_handler.go | 1 + service/history/history_engine_test.go | 1 + service/history/interfaces/mutable_state.go | 1 + .../history/workflow/mutable_state_impl.go | 4 ++++ ...utable_state_impl_restart_activity_test.go | 1 + .../workflow/mutable_state_impl_test.go | 2 ++ 12 files changed, 41 insertions(+), 17 deletions(-) diff --git a/api/persistence/v1/executions.pb.go b/api/persistence/v1/executions.pb.go index 2e6d4d5654..6c837e0ea2 100644 --- a/api/persistence/v1/executions.pb.go +++ b/api/persistence/v1/executions.pb.go @@ -2513,8 +2513,10 @@ type ActivityInfo struct { // set to true if reset heartbeat flag was set with an activity reset ResetHeartbeats bool `protobuf:"varint,48,opt,name=reset_heartbeats,json=resetHeartbeats,proto3" json:"reset_heartbeats,omitempty"` StartVersion int64 `protobuf:"varint,50,opt,name=start_version,json=startVersion,proto3" json:"start_version,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Worker instance key of the worker executing this activity. + WorkerInstanceKey string `protobuf:"bytes,51,opt,name=worker_instance_key,json=workerInstanceKey,proto3" json:"worker_instance_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActivityInfo) Reset() { @@ -2887,6 +2889,13 @@ func (x *ActivityInfo) GetStartVersion() int64 { return 0 } +func (x *ActivityInfo) GetWorkerInstanceKey() string { + if x != nil { + return x.WorkerInstanceKey + } + return "" +} + type isActivityInfo_BuildIdInfo interface { isActivityInfo_BuildIdInfo() } @@ -4831,7 +4840,7 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" + "\x17NexusInvocationTaskInfo\x12\x18\n" + "\aattempt\x18\x01 \x01(\x05R\aattempt\"4\n" + "\x18NexusCancelationTaskInfo\x12\x18\n" + - "\aattempt\x18\x01 \x01(\x05R\aattempt\"\x9e\x1b\n" + + "\aattempt\x18\x01 \x01(\x05R\aattempt\"\xce\x1b\n" + "\fActivityInfo\x12\x18\n" + "\aversion\x18\x01 \x01(\x03R\aversion\x127\n" + "\x18scheduled_event_batch_id\x18\x02 \x01(\x03R\x15scheduledEventBatchId\x12A\n" + @@ -4884,7 +4893,8 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" + "pause_info\x18. \x01(\v2:.temporal.server.api.persistence.v1.ActivityInfo.PauseInfoR\tpauseInfo\x12%\n" + "\x0eactivity_reset\x18/ \x01(\bR\ractivityReset\x12)\n" + "\x10reset_heartbeats\x180 \x01(\bR\x0fresetHeartbeats\x12#\n" + - "\rstart_version\x182 \x01(\x03R\fstartVersion\x1ay\n" + + "\rstart_version\x182 \x01(\x03R\fstartVersion\x12.\n" + + "\x13worker_instance_key\x183 \x01(\tR\x11workerInstanceKey\x1ay\n" + "\x16UseWorkflowBuildIdInfo\x12+\n" + "\x12last_used_build_id\x18\x01 \x01(\tR\x0flastUsedBuildId\x122\n" + "\x15last_redirect_counter\x18\x02 \x01(\x03R\x13lastRedirectCounter\x1a\x89\x02\n" + diff --git a/go.mod b/go.mod index 50abea769b..a610fa4807 100644 --- a/go.mod +++ b/go.mod @@ -171,3 +171,4 @@ require ( modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) + diff --git a/go.sum b/go.sum index e8cf371965..a4c698ef6a 100644 --- a/go.sum +++ b/go.sum @@ -373,8 +373,6 @@ go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.temporal.io/api v1.61.1-0.20260128230845-c246540cf2ed h1:g3CgsK5BXL2rQy0ZIJVRpNUDdtPM1y4bGv5ZoKsqR74= -go.temporal.io/api v1.61.1-0.20260128230845-c246540cf2ed/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= go.temporal.io/sdk v1.38.0 h1:4Bok5LEdED7YKpsSjIa3dDqram5VOq+ydBf4pyx0Wo4= go.temporal.io/sdk v1.38.0/go.mod h1:a+R2Ej28ObvHoILbHaxMyind7M6D+W0L7edt5UJF4SE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/proto/internal/temporal/server/api/persistence/v1/executions.proto b/proto/internal/temporal/server/api/persistence/v1/executions.proto index 116770bb82..f4c6351080 100644 --- a/proto/internal/temporal/server/api/persistence/v1/executions.proto +++ b/proto/internal/temporal/server/api/persistence/v1/executions.proto @@ -630,6 +630,9 @@ message ActivityInfo { bool reset_heartbeats = 48; int64 start_version = 50; + + // Worker instance key of the worker executing this activity. + string worker_instance_key = 51; } // timer_map column diff --git a/service/history/api/recordactivitytaskstarted/api.go b/service/history/api/recordactivitytaskstarted/api.go index 05c65076b7..381cbf94b5 100644 --- a/service/history/api/recordactivitytaskstarted/api.go +++ b/service/history/api/recordactivitytaskstarted/api.go @@ -242,6 +242,7 @@ func recordActivityTaskStarted( if _, err := mutableState.AddActivityTaskStartedEvent( ai, scheduledEventID, requestID, request.PollRequest.GetIdentity(), versioningStamp, pollerDeployment, request.GetBuildIdRedirectInfo(), + request.PollRequest.GetWorkerInstanceKey(), ); err != nil { return nil, rejectCodeUndefined, err } diff --git a/service/history/api/respondactivitytaskcompleted/api.go b/service/history/api/respondactivitytaskcompleted/api.go index 69764f9fd3..2186d0c2ba 100644 --- a/service/history/api/respondactivitytaskcompleted/api.go +++ b/service/history/api/respondactivitytaskcompleted/api.go @@ -88,17 +88,18 @@ func Invoke( // we need to force complete an activity fabricateStartedEvent = ai.StartedEventId == common.EmptyEventID if fabricateStartedEvent { - _, err := mutableState.AddActivityTaskStartedEvent( - ai, - scheduledEventID, - "", - req.GetCompleteRequest().GetIdentity(), - nil, - nil, - // TODO (shahab): do we need to do anything with wf redirect in this case or any - // other case where an activity starts? - nil, - ) + _, err := mutableState.AddActivityTaskStartedEvent( + ai, + scheduledEventID, + "", + req.GetCompleteRequest().GetIdentity(), + nil, + nil, + // TODO (shahab): do we need to do anything with wf redirect in this case or any + // other case where an activity starts? + nil, + "", // workerInstanceKey not available for force complete + ) if err != nil { return nil, err } diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index e58ada9289..fb6fa7142e 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -556,6 +556,7 @@ func (handler *workflowTaskCompletedHandler) handlePostCommandEagerExecuteActivi stamp, nil, nil, + "", // workerInstanceKey not available for eager dispatch ); err != nil { return nil, err } diff --git a/service/history/history_engine_test.go b/service/history/history_engine_test.go index 7ce8a939b0..407f508c83 100644 --- a/service/history/history_engine_test.go +++ b/service/history/history_engine_test.go @@ -6582,6 +6582,7 @@ func addActivityTaskStartedEvent(ms historyi.MutableState, scheduledEventID int6 nil, nil, nil, + "", ) return event } diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index daec80d476..816c0ee838 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -57,6 +57,7 @@ type ( *commonpb.WorkerVersionStamp, *deploymentpb.Deployment, *taskqueuespb.BuildIdRedirectInfo, + string, // workerInstanceKey ) (*historypb.HistoryEvent, error) AddActivityTaskTimedOutEvent(int64, int64, *failurepb.Failure, enumspb.RetryState) (*historypb.HistoryEvent, error) AddChildWorkflowExecutionCanceledEvent(int64, *commonpb.WorkflowExecution, *historypb.WorkflowExecutionCanceledEventAttributes) (*historypb.HistoryEvent, error) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index af50fccc29..03b833bad0 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -4037,6 +4037,7 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent( versioningStamp *commonpb.WorkerVersionStamp, deployment *deploymentpb.Deployment, redirectInfo *taskqueuespb.BuildIdRedirectInfo, + workerInstanceKey string, ) (*historypb.HistoryEvent, error) { opTag := tag.WorkflowActionActivityTaskStarted err := ms.checkMutability(opTag) @@ -4065,6 +4066,8 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent( ai.LastDeploymentVersion = worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(deployment) } + ai.WorkerInstanceKey = workerInstanceKey + if !ai.HasRetryPolicy { event := ms.hBuilder.AddActivityTaskStartedEvent( scheduledEventID, @@ -4096,6 +4099,7 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent( activityInfo.RequestId = requestID activityInfo.StartedTime = timestamppb.New(ms.timeSource.Now()) activityInfo.StartedIdentity = identity + activityInfo.WorkerInstanceKey = workerInstanceKey return nil }); err != nil { return nil, err diff --git a/service/history/workflow/mutable_state_impl_restart_activity_test.go b/service/history/workflow/mutable_state_impl_restart_activity_test.go index 9337a98fff..3cb91b7fbd 100644 --- a/service/history/workflow/mutable_state_impl_restart_activity_test.go +++ b/service/history/workflow/mutable_state_impl_restart_activity_test.go @@ -419,6 +419,7 @@ func (s *retryActivitySuite) makeActivityAndPutIntoFailingState() *persistencesp nil, nil, nil, + "", ) s.NoError(err) diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 3af55e0066..72163aedb8 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -2875,6 +2875,7 @@ func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() { nil, nil, nil, + "", ) s.NoError(err) @@ -2940,6 +2941,7 @@ func (s *mutableStateSuite) TestRetryActivity_PausedIncrementsStamp() { nil, nil, nil, + "", ) s.NoError(err) From db7ef5c8f57be467c3a1c167e57c58c04b15d672 Mon Sep 17 00:00:00 2001 From: Kannan Rajah Date: Thu, 5 Feb 2026 09:53:22 -0800 Subject: [PATCH 2/3] Add unit test --- api/persistence/v1/executions.pb.go | 2 +- go.mod | 1 - go.sum | 2 + .../api/persistence/v1/executions.proto | 2 +- .../history/interfaces/mutable_state_mock.go | 8 +- .../workflow/mutable_state_impl_test.go | 73 +++++++++++++++++++ 6 files changed, 81 insertions(+), 7 deletions(-) diff --git a/api/persistence/v1/executions.pb.go b/api/persistence/v1/executions.pb.go index 6c837e0ea2..ce2420628d 100644 --- a/api/persistence/v1/executions.pb.go +++ b/api/persistence/v1/executions.pb.go @@ -2513,7 +2513,7 @@ type ActivityInfo struct { // set to true if reset heartbeat flag was set with an activity reset ResetHeartbeats bool `protobuf:"varint,48,opt,name=reset_heartbeats,json=resetHeartbeats,proto3" json:"reset_heartbeats,omitempty"` StartVersion int64 `protobuf:"varint,50,opt,name=start_version,json=startVersion,proto3" json:"start_version,omitempty"` - // Worker instance key of the worker executing this activity. + // Unique identifier of the worker that is this activity. WorkerInstanceKey string `protobuf:"bytes,51,opt,name=worker_instance_key,json=workerInstanceKey,proto3" json:"worker_instance_key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache diff --git a/go.mod b/go.mod index a610fa4807..50abea769b 100644 --- a/go.mod +++ b/go.mod @@ -171,4 +171,3 @@ require ( modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) - diff --git a/go.sum b/go.sum index a4c698ef6a..e8cf371965 100644 --- a/go.sum +++ b/go.sum @@ -373,6 +373,8 @@ go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.temporal.io/api v1.61.1-0.20260128230845-c246540cf2ed h1:g3CgsK5BXL2rQy0ZIJVRpNUDdtPM1y4bGv5ZoKsqR74= +go.temporal.io/api v1.61.1-0.20260128230845-c246540cf2ed/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= go.temporal.io/sdk v1.38.0 h1:4Bok5LEdED7YKpsSjIa3dDqram5VOq+ydBf4pyx0Wo4= go.temporal.io/sdk v1.38.0/go.mod h1:a+R2Ej28ObvHoILbHaxMyind7M6D+W0L7edt5UJF4SE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/proto/internal/temporal/server/api/persistence/v1/executions.proto b/proto/internal/temporal/server/api/persistence/v1/executions.proto index f4c6351080..77e34189a4 100644 --- a/proto/internal/temporal/server/api/persistence/v1/executions.proto +++ b/proto/internal/temporal/server/api/persistence/v1/executions.proto @@ -631,7 +631,7 @@ message ActivityInfo { int64 start_version = 50; - // Worker instance key of the worker executing this activity. + // Unique identifier of the worker that is this activity. string worker_instance_key = 51; } diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go index c6ce89b78d..8b6d8a803c 100644 --- a/service/history/interfaces/mutable_state_mock.go +++ b/service/history/interfaces/mutable_state_mock.go @@ -148,18 +148,18 @@ func (mr *MockMutableStateMockRecorder) AddActivityTaskScheduledEvent(arg0, arg1 } // AddActivityTaskStartedEvent mocks base method. -func (m *MockMutableState) AddActivityTaskStartedEvent(arg0 *persistence.ActivityInfo, arg1 int64, arg2, arg3 string, arg4 *common.WorkerVersionStamp, arg5 *deployment.Deployment, arg6 *taskqueue0.BuildIdRedirectInfo) (*history.HistoryEvent, error) { +func (m *MockMutableState) AddActivityTaskStartedEvent(arg0 *persistence.ActivityInfo, arg1 int64, arg2, arg3 string, arg4 *common.WorkerVersionStamp, arg5 *deployment.Deployment, arg6 *taskqueue0.BuildIdRedirectInfo, arg7 string) (*history.HistoryEvent, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddActivityTaskStartedEvent", arg0, arg1, arg2, arg3, arg4, arg5, arg6) + ret := m.ctrl.Call(m, "AddActivityTaskStartedEvent", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) ret0, _ := ret[0].(*history.HistoryEvent) ret1, _ := ret[1].(error) return ret0, ret1 } // AddActivityTaskStartedEvent indicates an expected call of AddActivityTaskStartedEvent. -func (mr *MockMutableStateMockRecorder) AddActivityTaskStartedEvent(arg0, arg1, arg2, arg3, arg4, arg5, arg6 any) *gomock.Call { +func (mr *MockMutableStateMockRecorder) AddActivityTaskStartedEvent(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddActivityTaskStartedEvent", reflect.TypeOf((*MockMutableState)(nil).AddActivityTaskStartedEvent), arg0, arg1, arg2, arg3, arg4, arg5, arg6) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddActivityTaskStartedEvent", reflect.TypeOf((*MockMutableState)(nil).AddActivityTaskStartedEvent), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) } // AddActivityTaskTimedOutEvent mocks base method. diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 72163aedb8..7de5e4d4af 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -6112,3 +6112,76 @@ func (s *mutableStateSuite) TestCHASMNodeSize() { expectedTotalSize += len(newNodeKey) + newNode.Size() s.Equal(expectedTotalSize, mutableState.GetApproximatePersistedSize()) } + +func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerInstanceKey() { + s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() + + // Setup workflow execution + _, err := s.mutableState.AddWorkflowExecutionStartedEvent( + &commonpb.WorkflowExecution{WorkflowId: tests.WorkflowID, RunId: tests.RunID}, + &historyservice.StartWorkflowExecutionRequest{ + NamespaceId: tests.NamespaceID.String(), + StartRequest: &workflowservice.StartWorkflowExecutionRequest{ + WorkflowType: &commonpb.WorkflowType{Name: "workflow-type"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "task-queue"}, + WorkflowRunTimeout: durationpb.New(200 * time.Second), + WorkflowTaskTimeout: durationpb.New(1 * time.Second), + }, + }, + ) + s.NoError(err) + + di, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + _, _, err = s.mutableState.AddWorkflowTaskStartedEvent( + di.ScheduledEventID, + di.RequestID, + di.TaskQueue, + "identity", + nil, + nil, + nil, + false, + nil, + ) + s.NoError(err) + _, err = s.mutableState.AddWorkflowTaskCompletedEvent( + di, + &workflowservice.RespondWorkflowTaskCompletedRequest{Identity: "identity"}, + workflowTaskCompletionLimits, + ) + s.NoError(err) + + // Schedule activity + workflowTaskCompletedEventID := int64(4) + _, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent( + workflowTaskCompletedEventID, + &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: "test-activity-1", + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + }, + false, + ) + s.NoError(err) + s.Empty(activityInfo.WorkerInstanceKey, "WorkerInstanceKey should be empty before activity starts") + + // Start activity with workerInstanceKey + expectedWorkerInstanceKey := "test-worker-instance-key-12345" + _, err = s.mutableState.AddActivityTaskStartedEvent( + activityInfo, + activityInfo.ScheduledEventId, + uuid.NewString(), + "worker-identity", + nil, + nil, + nil, + expectedWorkerInstanceKey, + ) + s.NoError(err) + + // Verify workerInstanceKey is stored + updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) + s.True(ok) + s.Equal(expectedWorkerInstanceKey, updatedActivityInfo.WorkerInstanceKey) +} From e66ae7e90b244e299b72be01cdee6c67b5c33bc7 Mon Sep 17 00:00:00 2001 From: Kannan Rajah Date: Thu, 5 Feb 2026 13:16:54 -0800 Subject: [PATCH 3/3] Fix lint --- .../api/respondactivitytaskcompleted/api.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/service/history/api/respondactivitytaskcompleted/api.go b/service/history/api/respondactivitytaskcompleted/api.go index 2186d0c2ba..d9c5a8d665 100644 --- a/service/history/api/respondactivitytaskcompleted/api.go +++ b/service/history/api/respondactivitytaskcompleted/api.go @@ -88,18 +88,18 @@ func Invoke( // we need to force complete an activity fabricateStartedEvent = ai.StartedEventId == common.EmptyEventID if fabricateStartedEvent { - _, err := mutableState.AddActivityTaskStartedEvent( - ai, - scheduledEventID, - "", - req.GetCompleteRequest().GetIdentity(), - nil, - nil, - // TODO (shahab): do we need to do anything with wf redirect in this case or any - // other case where an activity starts? - nil, - "", // workerInstanceKey not available for force complete - ) + _, err := mutableState.AddActivityTaskStartedEvent( + ai, + scheduledEventID, + "", + req.GetCompleteRequest().GetIdentity(), + nil, + nil, + // TODO (shahab): do we need to do anything with wf redirect in this case or any + // other case where an activity starts? + nil, + "", // workerInstanceKey not available for force complete + ) if err != nil { return nil, err }