Skip to content

Store worker_instance_key for an activity in mutable state #9231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rkannan82 wants to merge 3 commits into
base: main
Choose a base branch
from
+106 −8
Open

Store worker_instance_key for an activity in mutable state #9231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
07a4d60
Store worker_instance_key in ActivityInfo
rkannan82 Feb 4, 2026
db7ef5c
Add unit test
rkannan82 Feb 5, 2026
e66ae7e
Fix lint
rkannan82 Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions api/persistence/v1/executions.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions proto/internal/temporal/server/api/persistence/v1/executions.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -630,6 +630,9 @@ message ActivityInfo {
bool reset_heartbeats = 48;

int64 start_version = 50;

// Unique identifier of the worker that is this activity.
string worker_instance_key = 51;
}

// timer_map column
Expand Down
1 change: 1 addition & 0 deletions service/history/api/recordactivitytaskstarted/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ func recordActivityTaskStarted(
if _, err := mutableState.AddActivityTaskStartedEvent(
ai, scheduledEventID, requestID, request.PollRequest.GetIdentity(),
versioningStamp, pollerDeployment, request.GetBuildIdRedirectInfo(),
request.PollRequest.GetWorkerInstanceKey(),
); err != nil {
return nil, rejectCodeUndefined, err
}
Expand Down
1 change: 1 addition & 0 deletions service/history/api/respondactivitytaskcompleted/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ func Invoke(
// TODO (shahab): do we need to do anything with wf redirect in this case or any
// other case where an activity starts?
nil,
"", // workerInstanceKey not available for force complete
)
if err != nil {
return nil, err
Expand Down
1 change: 1 addition & 0 deletions service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,7 @@ func (handler *workflowTaskCompletedHandler) handlePostCommandEagerExecuteActivi
stamp,
nil,
nil,
"", // workerInstanceKey not available for eager dispatch
); err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions service/history/history_engine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6582,6 +6582,7 @@ func addActivityTaskStartedEvent(ms historyi.MutableState, scheduledEventID int6
nil,
nil,
nil,
"",
)
return event
}
Expand Down
1 change: 1 addition & 0 deletions service/history/interfaces/mutable_state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ type (
*commonpb.WorkerVersionStamp,
*deploymentpb.Deployment,
*taskqueuespb.BuildIdRedirectInfo,
string, // workerInstanceKey
) (*historypb.HistoryEvent, error)
AddActivityTaskTimedOutEvent(int64, int64, *failurepb.Failure, enumspb.RetryState) (*historypb.HistoryEvent, error)
AddChildWorkflowExecutionCanceledEvent(int64, *commonpb.WorkflowExecution, *historypb.WorkflowExecutionCanceledEventAttributes) (*historypb.HistoryEvent, error)
Expand Down
8 changes: 4 additions & 4 deletions service/history/interfaces/mutable_state_mock.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions service/history/workflow/mutable_state_impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4037,6 +4037,7 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent(
versioningStamp *commonpb.WorkerVersionStamp,
deployment *deploymentpb.Deployment,
redirectInfo *taskqueuespb.BuildIdRedirectInfo,
workerInstanceKey string,
) (*historypb.HistoryEvent, error) {
opTag := tag.WorkflowActionActivityTaskStarted
err := ms.checkMutability(opTag)
Expand Down Expand Up @@ -4065,6 +4066,8 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent(
ai.LastDeploymentVersion = worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(deployment)
}

ai.WorkerInstanceKey = workerInstanceKey

if !ai.HasRetryPolicy {
event := ms.hBuilder.AddActivityTaskStartedEvent(
scheduledEventID,
Expand Down Expand Up @@ -4096,6 +4099,7 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent(
activityInfo.RequestId = requestID
activityInfo.StartedTime = timestamppb.New(ms.timeSource.Now())
activityInfo.StartedIdentity = identity
activityInfo.WorkerInstanceKey = workerInstanceKey
return nil
}); err != nil {
return nil, err
Expand Down
1 change: 1 addition & 0 deletions service/history/workflow/mutable_state_impl_restart_activity_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,7 @@ func (s *retryActivitySuite) makeActivityAndPutIntoFailingState() *persistencesp
nil,
nil,
nil,
"",
)
s.NoError(err)

Expand Down
75 changes: 75 additions & 0 deletions service/history/workflow/mutable_state_impl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2875,6 +2875,7 @@ func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() {
nil,
nil,
nil,
"",
)
s.NoError(err)

Expand Down Expand Up @@ -2940,6 +2941,7 @@ func (s *mutableStateSuite) TestRetryActivity_PausedIncrementsStamp() {
nil,
nil,
nil,
"",
)
s.NoError(err)

Expand Down Expand Up @@ -6110,3 +6112,76 @@ func (s *mutableStateSuite) TestCHASMNodeSize() {
expectedTotalSize += len(newNodeKey) + newNode.Size()
s.Equal(expectedTotalSize, mutableState.GetApproximatePersistedSize())
}

func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerInstanceKey() {
s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes()

// Setup workflow execution
_, err := s.mutableState.AddWorkflowExecutionStartedEvent(
&commonpb.WorkflowExecution{WorkflowId: tests.WorkflowID, RunId: tests.RunID},
&historyservice.StartWorkflowExecutionRequest{
NamespaceId: tests.NamespaceID.String(),
StartRequest: &workflowservice.StartWorkflowExecutionRequest{
WorkflowType: &commonpb.WorkflowType{Name: "workflow-type"},
TaskQueue: &taskqueuepb.TaskQueue{Name: "task-queue"},
WorkflowRunTimeout: durationpb.New(200 * time.Second),
WorkflowTaskTimeout: durationpb.New(1 * time.Second),
},
},
)
s.NoError(err)

di, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL)
s.NoError(err)
_, _, err = s.mutableState.AddWorkflowTaskStartedEvent(
di.ScheduledEventID,
di.RequestID,
di.TaskQueue,
"identity",
nil,
nil,
nil,
false,
nil,
)
s.NoError(err)
_, err = s.mutableState.AddWorkflowTaskCompletedEvent(
di,
&workflowservice.RespondWorkflowTaskCompletedRequest{Identity: "identity"},
workflowTaskCompletionLimits,
)
s.NoError(err)

// Schedule activity
workflowTaskCompletedEventID := int64(4)
_, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent(
workflowTaskCompletedEventID,
&commandpb.ScheduleActivityTaskCommandAttributes{
ActivityId: "test-activity-1",
ActivityType: &commonpb.ActivityType{Name: "test-activity-type"},
TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"},
},
false,
)
s.NoError(err)
s.Empty(activityInfo.WorkerInstanceKey, "WorkerInstanceKey should be empty before activity starts")

// Start activity with workerInstanceKey
expectedWorkerInstanceKey := "test-worker-instance-key-12345"
_, err = s.mutableState.AddActivityTaskStartedEvent(
activityInfo,
activityInfo.ScheduledEventId,
uuid.NewString(),
"worker-identity",
nil,
nil,
nil,
expectedWorkerInstanceKey,
)
s.NoError(err)

// Verify workerInstanceKey is stored
updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId)
s.True(ok)
s.Equal(expectedWorkerInstanceKey, updatedActivityInfo.WorkerInstanceKey)
}
Loading