Increased timeout value for delete history event task (#5888)

* Increased timeout value for delete history event task to dynamic config with default at 30s
uber · Apr 9, 2024 · cd14f34 · cd14f34
1 parent 65c4017
commit cd14f34
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 34 deletions.
diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go
@@ -1440,6 +1440,13 @@ const (
 	IsolationGroupStateUpdateRetryAttempts
 
 	LargeShardHistoryBlobMetricThreshold
+
+	// DeleteHistoryEventContextTimeout in seconds
+	// KeyName: system.deleteHistoryEventContextTimeout
+	// Value type: Int
+	// Default value: 30
+	DeleteHistoryEventContextTimeout
+
 	// LastIntKey must be the last one in this const group
 	LastIntKey
 )
@@ -3791,6 +3798,11 @@ var IntKeys = map[IntKey]DynamicInt{
 		Description:  "The number of attempts to push Isolation group configuration to the config store",
 		DefaultValue: 2,
 	},
+	DeleteHistoryEventContextTimeout: DynamicInt{
+		KeyName:      "system.deleteHistoryEventContextTimeout",
+		Description:  "This is the number of seconds allowed for a deleteHistoryEvent task to the database",
+		DefaultValue: 30,
+	},
 }
 
 var BoolKeys = map[BoolKey]DynamicBool{

diff --git a/service/history/config/config.go b/service/history/config/config.go
@@ -31,35 +31,36 @@ import (
 
 // Config represents configuration for cadence-history service
 type Config struct {
-	NumberOfShards                  int
-	IsAdvancedVisConfigExist        bool
-	RPS                             dynamicconfig.IntPropertyFn
-	MaxIDLengthWarnLimit            dynamicconfig.IntPropertyFn
-	DomainNameMaxLength             dynamicconfig.IntPropertyFnWithDomainFilter
-	IdentityMaxLength               dynamicconfig.IntPropertyFnWithDomainFilter
-	WorkflowIDMaxLength             dynamicconfig.IntPropertyFnWithDomainFilter
-	SignalNameMaxLength             dynamicconfig.IntPropertyFnWithDomainFilter
-	WorkflowTypeMaxLength           dynamicconfig.IntPropertyFnWithDomainFilter
-	RequestIDMaxLength              dynamicconfig.IntPropertyFnWithDomainFilter
-	TaskListNameMaxLength           dynamicconfig.IntPropertyFnWithDomainFilter
-	ActivityIDMaxLength             dynamicconfig.IntPropertyFnWithDomainFilter
-	ActivityTypeMaxLength           dynamicconfig.IntPropertyFnWithDomainFilter
-	MarkerNameMaxLength             dynamicconfig.IntPropertyFnWithDomainFilter
-	TimerIDMaxLength                dynamicconfig.IntPropertyFnWithDomainFilter
-	PersistenceMaxQPS               dynamicconfig.IntPropertyFn
-	PersistenceGlobalMaxQPS         dynamicconfig.IntPropertyFn
-	EnableVisibilitySampling        dynamicconfig.BoolPropertyFn
-	EnableReadFromClosedExecutionV2 dynamicconfig.BoolPropertyFn
-	VisibilityOpenMaxQPS            dynamicconfig.IntPropertyFnWithDomainFilter
-	VisibilityClosedMaxQPS          dynamicconfig.IntPropertyFnWithDomainFilter
-	AdvancedVisibilityWritingMode   dynamicconfig.StringPropertyFn
-	EmitShardDiffLog                dynamicconfig.BoolPropertyFn
-	MaxAutoResetPoints              dynamicconfig.IntPropertyFnWithDomainFilter
-	ThrottledLogRPS                 dynamicconfig.IntPropertyFn
-	EnableStickyQuery               dynamicconfig.BoolPropertyFnWithDomainFilter
-	ShutdownDrainDuration           dynamicconfig.DurationPropertyFn
-	WorkflowDeletionJitterRange     dynamicconfig.IntPropertyFnWithDomainFilter
-	MaxResponseSize                 int
+	NumberOfShards                   int
+	IsAdvancedVisConfigExist         bool
+	RPS                              dynamicconfig.IntPropertyFn
+	MaxIDLengthWarnLimit             dynamicconfig.IntPropertyFn
+	DomainNameMaxLength              dynamicconfig.IntPropertyFnWithDomainFilter
+	IdentityMaxLength                dynamicconfig.IntPropertyFnWithDomainFilter
+	WorkflowIDMaxLength              dynamicconfig.IntPropertyFnWithDomainFilter
+	SignalNameMaxLength              dynamicconfig.IntPropertyFnWithDomainFilter
+	WorkflowTypeMaxLength            dynamicconfig.IntPropertyFnWithDomainFilter
+	RequestIDMaxLength               dynamicconfig.IntPropertyFnWithDomainFilter
+	TaskListNameMaxLength            dynamicconfig.IntPropertyFnWithDomainFilter
+	ActivityIDMaxLength              dynamicconfig.IntPropertyFnWithDomainFilter
+	ActivityTypeMaxLength            dynamicconfig.IntPropertyFnWithDomainFilter
+	MarkerNameMaxLength              dynamicconfig.IntPropertyFnWithDomainFilter
+	TimerIDMaxLength                 dynamicconfig.IntPropertyFnWithDomainFilter
+	PersistenceMaxQPS                dynamicconfig.IntPropertyFn
+	PersistenceGlobalMaxQPS          dynamicconfig.IntPropertyFn
+	EnableVisibilitySampling         dynamicconfig.BoolPropertyFn
+	EnableReadFromClosedExecutionV2  dynamicconfig.BoolPropertyFn
+	VisibilityOpenMaxQPS             dynamicconfig.IntPropertyFnWithDomainFilter
+	VisibilityClosedMaxQPS           dynamicconfig.IntPropertyFnWithDomainFilter
+	AdvancedVisibilityWritingMode    dynamicconfig.StringPropertyFn
+	EmitShardDiffLog                 dynamicconfig.BoolPropertyFn
+	MaxAutoResetPoints               dynamicconfig.IntPropertyFnWithDomainFilter
+	ThrottledLogRPS                  dynamicconfig.IntPropertyFn
+	EnableStickyQuery                dynamicconfig.BoolPropertyFnWithDomainFilter
+	ShutdownDrainDuration            dynamicconfig.DurationPropertyFn
+	WorkflowDeletionJitterRange      dynamicconfig.IntPropertyFnWithDomainFilter
+	DeleteHistoryEventContextTimeout dynamicconfig.IntPropertyFn
+	MaxResponseSize                  int
 
 	// HistoryCache settings
 	// Change of these configs require shard restart
@@ -387,6 +388,7 @@ func New(dc *dynamicconfig.Collection, numberOfShards int, maxMessageSize int, s
 		StandbyTaskMissingEventsResendDelay:  dc.GetDurationProperty(dynamicconfig.StandbyTaskMissingEventsResendDelay),
 		StandbyTaskMissingEventsDiscardDelay: dc.GetDurationProperty(dynamicconfig.StandbyTaskMissingEventsDiscardDelay),
 		WorkflowDeletionJitterRange:          dc.GetIntPropertyFilteredByDomain(dynamicconfig.WorkflowDeletionJitterRange),
+		DeleteHistoryEventContextTimeout:     dc.GetIntProperty(dynamicconfig.DeleteHistoryEventContextTimeout),
 		MaxResponseSize:                      maxMessageSize,
 
 		TaskProcessRPS:                          dc.GetIntPropertyFilteredByDomain(dynamicconfig.TaskProcessRPS),

diff --git a/service/history/task/timer_active_task_executor.go b/service/history/task/timer_active_task_executor.go
@@ -104,7 +104,10 @@ func (t *timerActiveTaskExecutor) Execute(
 	case persistence.TaskTypeWorkflowBackoffTimer:
 		return t.executeWorkflowBackoffTimerTask(ctx, timerTask)
 	case persistence.TaskTypeDeleteHistoryEvent:
-		return t.executeDeleteHistoryEventTask(ctx, timerTask)
+		// special timeout for delete history event
+		deleteHistoryEventContext, deleteHistoryEventCancel := context.WithTimeout(context.Background(), time.Duration(t.config.DeleteHistoryEventContextTimeout())*time.Second)
+		defer deleteHistoryEventCancel()
+		return t.executeDeleteHistoryEventTask(deleteHistoryEventContext, timerTask)
 	default:
 		return errUnknownTimerTask
 	}

diff --git a/service/history/task/timer_active_task_executor_test.go b/service/history/task/timer_active_task_executor_test.go
@@ -1246,3 +1246,14 @@ func (s *timerActiveTaskExecutorSuite) newTimerTaskFromInfo(
 ) Task {
 	return NewTimerTask(s.mockShard, info, QueueTypeActiveTimer, s.logger, nil, nil, nil, nil, nil)
 }
+
+func (s *timerActiveTaskExecutorSuite) TestActiveTaskTimeout() {
+	deleteHistoryEventTask := s.newTimerTaskFromInfo(&persistence.TimerTaskInfo{
+		Version:     s.version,
+		DomainID:    s.domainID,
+		TaskID:      int64(100),
+		TaskType:    persistence.TaskTypeDeleteHistoryEvent,
+		TimeoutType: int(types.TimeoutTypeStartToClose),
+	})
+	s.timerActiveTaskExecutor.Execute(deleteHistoryEventTask, true)
+}
diff --git a/service/history/task/timer_standby_task_executor.go b/service/history/task/timer_standby_task_executor.go
@@ -110,7 +110,10 @@ func (t *timerStandbyTaskExecutor) Execute(
 	case persistence.TaskTypeWorkflowBackoffTimer:
 		return t.executeWorkflowBackoffTimerTask(ctx, timerTask)
 	case persistence.TaskTypeDeleteHistoryEvent:
-		return t.executeDeleteHistoryEventTask(ctx, timerTask)
+		// special timeout for delete history event
+		deleteHistoryEventContext, deleteHistoryEventCancel := context.WithTimeout(context.Background(), time.Duration(t.config.DeleteHistoryEventContextTimeout())*time.Second)
+		defer deleteHistoryEventCancel()
+		return t.executeDeleteHistoryEventTask(deleteHistoryEventContext, timerTask)
 	default:
 		return errUnknownTimerTask
 	}

diff --git a/service/history/task/timer_standby_task_executor_test.go b/service/history/task/timer_standby_task_executor_test.go
@@ -808,3 +808,14 @@ func (s *timerStandbyTaskExecutorSuite) newTimerTaskFromInfo(
 ) Task {
 	return NewTimerTask(s.mockShard, info, QueueTypeStandbyTimer, s.logger, nil, nil, nil, nil, nil)
 }
+
+func (s *timerStandbyTaskExecutorSuite) TestTransferTaskTimeout() {
+	deleteHistoryEventTask := s.newTimerTaskFromInfo(&persistence.TimerTaskInfo{
+		Version:     s.version,
+		DomainID:    s.domainID,
+		TaskID:      int64(100),
+		TaskType:    persistence.TaskTypeDeleteHistoryEvent,
+		TimeoutType: int(types.TimeoutTypeStartToClose),
+	})
+	s.timerStandbyTaskExecutor.Execute(deleteHistoryEventTask, true)
+}
diff --git a/service/history/task/transfer_task_executor_base.go b/service/history/task/transfer_task_executor_base.go
@@ -43,9 +43,8 @@ const (
 	taskDefaultTimeout             = 3 * time.Second
 	taskGetExecutionContextTimeout = 500 * time.Millisecond
 	taskRPCCallTimeout             = 2 * time.Second
-
-	secondsInDay      = int32(24 * time.Hour / time.Second)
-	defaultDomainName = "defaultDomainName"
+	secondsInDay                   = int32(24 * time.Hour / time.Second)
+	defaultDomainName              = "defaultDomainName"
 )
 
 type (