Do not blindly retry timer task read (#2500)

* Do not blindly retry timer task read, rely on timer controller rate limiter instead

common/persistence/nosql/nosqlplugin/cassandra/gocql/errors.go

@@ -63,28 +63,6 @@ func ConvertError(
 	}
 }
 
-func IsTimeoutError(err error) bool {
-	if err == context.DeadlineExceeded {
-		return true
-	}
-	if err == gocql.ErrTimeoutNoResponse {
-		return true
-	}
-	if err == gocql.ErrConnectionClosed {
-		return true
-	}
-	_, ok := err.(*gocql.RequestErrWriteTimeout)
-	return ok
-}
-
 func IsNotFoundError(err error) bool {
 	return err == gocql.ErrNotFound
 }
-
-func IsThrottlingError(err error) bool {
-	if req, ok := err.(gocql.RequestError); ok {
-		// gocql does not expose the constant errOverloaded = 0x1001
-		return req.Code() == 0x1001
-	}
-	return false
-}

service/history/timerQueueAckMgr.go

@@ -32,7 +32,6 @@ import (
 
 	"go.temporal.io/server/common/backoff"
 	"go.temporal.io/server/service/history/configs"
-	"go.temporal.io/server/service/history/consts"
 	"go.temporal.io/server/service/history/shard"
 	"go.temporal.io/server/service/history/tasks"
 
@@ -44,8 +43,6 @@ import (
 
 var (
 	maximumTime = time.Unix(0, math.MaxInt64).UTC()
-
-	timerRetryPolicy = createTimerRetryPolicy()
 )
 
 const (
@@ -415,19 +412,9 @@ func (t *timerQueueAckMgrImpl) getTimerTasks(minTimestamp time.Time, maxTimestam
 		BatchSize:     batchSize,
 		NextPageToken: pageToken,
 	}
-
-	var response *persistence.GetTimerTasksResponse
-	var err error
-	op := func() error {
-		response, err = t.executionMgr.GetTimerTasks(request)
-		return err
-	}
-
-	err = backoff.Retry(op, timerRetryPolicy, func(err error) bool {
-		return true
-	})
+	response, err := t.executionMgr.GetTimerTasks(request)
 	if err != nil {
-		return nil, nil, consts.ErrMaxAttemptsExceeded
+		return nil, nil, err
 	}
 	return response.Tasks, response.NextPageToken, nil
 }

service/history/transferQueueProcessorBase.go

@@ -79,11 +79,9 @@ func (t *transferQueueProcessorBase) readTasks(
 		MaxReadLevel: t.maxReadAckLevel(),
 		BatchSize:    t.options.BatchSize(),
 	})
-
 	if err != nil {
 		return nil, false, err
 	}
-
 	return response.Tasks, len(response.NextPageToken) != 0, nil
 }