diff --git a/pkg/retry/errors.go b/pkg/retry/errors.go index e0348968a4..1b075efb27 100644 --- a/pkg/retry/errors.go +++ b/pkg/retry/errors.go @@ -45,6 +45,12 @@ var ( "binlog checksum mismatch, data may be corrupted", "get event err EOF", } + + // Retryable1105Msgs list the error messages of some retryable error with 1105 code. + Retryable1105Msgs = []string{ + "Information schema is out of date", + "Information schema is changed", + } ) // IsRetryableError tells whether this error should retry @@ -54,8 +60,14 @@ func IsRetryableError(err error) bool { if ok { switch mysqlErr.Number { // ER_LOCK_DEADLOCK can retry to commit while meet deadlock - case tmysql.ErrUnknown, gmysql.ER_LOCK_DEADLOCK, tmysql.ErrPDServerTimeout, tmysql.ErrTiKVServerTimeout, tmysql.ErrTiKVServerBusy, tmysql.ErrResolveLockTimeout, tmysql.ErrRegionUnavailable, tmysql.ErrQueryInterrupted, tmysql.ErrWriteConflictInTiDB, tmysql.ErrTableLocked, tmysql.ErrWriteConflict: + case gmysql.ER_LOCK_DEADLOCK, tmysql.ErrPDServerTimeout, tmysql.ErrTiKVServerTimeout, tmysql.ErrTiKVServerBusy, tmysql.ErrResolveLockTimeout, tmysql.ErrRegionUnavailable, tmysql.ErrQueryInterrupted, tmysql.ErrWriteConflictInTiDB, tmysql.ErrTableLocked, tmysql.ErrWriteConflict: return true + case tmysql.ErrUnknown: + for _, msg := range Retryable1105Msgs { + if strings.Contains(mysqlErr.Message, msg) { + return true + } + } default: return false } @@ -72,18 +84,3 @@ func IsConnectionError(err error) bool { } return false } - -// IsRetryableErrorFastFailFilter tells whether this error should retry, -// filtering some incompatible DDL error to achieve fast fail. -func IsRetryableErrorFastFailFilter(err error) bool { - err2 := errors.Cause(err) // check the original error - if mysqlErr, ok := err2.(*mysql.MySQLError); ok && mysqlErr.Number == tmysql.ErrUnknown { - for _, msg := range UnsupportedDDLMsgs { - if strings.Contains(mysqlErr.Message, msg) { - return false - } - } - } - - return IsRetryableError(err) -} diff --git a/syncer/db.go b/syncer/db.go index 85ca95034c..7031366913 100644 --- a/syncer/db.go +++ b/syncer/db.go @@ -238,7 +238,7 @@ func (conn *DBConn) executeSQLWithIgnore(tctx *tcontext.Context, ignoreError fun sqlRetriesTotal.WithLabelValues("stmt_exec", conn.cfg.Name).Add(1) return true } - if retry.IsRetryableErrorFastFailFilter(err) { + if retry.IsRetryableError(err) { tctx.L().Warn("execute statements", zap.Int("retry", retryTime), zap.String("queries", utils.TruncateInterface(queries, -1)), zap.String("arguments", utils.TruncateInterface(args, -1))) diff --git a/syncer/error_test.go b/syncer/error_test.go index ed00eff6f1..e7131f672a 100644 --- a/syncer/error_test.go +++ b/syncer/error_test.go @@ -40,7 +40,6 @@ func (s *testSyncerSuite) TestIsRetryableError(c *C) { }{ {newMysqlErr(tmysql.ErrNoDB, "no baseConn error"), false}, {errors.New("unknown error"), false}, - {newMysqlErr(tmysql.ErrUnknown, "i/o timeout"), true}, {newMysqlErr(tmysql.ErrDBCreateExists, "baseConn already exists"), false}, {driver.ErrBadConn, false}, {newMysqlErr(gmysql.ER_LOCK_DEADLOCK, "Deadlock found when trying to get lock; try restarting transaction"), true}, @@ -49,6 +48,10 @@ func (s *testSyncerSuite) TestIsRetryableError(c *C) { {newMysqlErr(tmysql.ErrTiKVServerBusy, "tikv server busy"), true}, {newMysqlErr(tmysql.ErrResolveLockTimeout, "resolve lock timeout"), true}, {newMysqlErr(tmysql.ErrRegionUnavailable, "region unavailable"), true}, + {newMysqlErr(tmysql.ErrUnknown, "i/o timeout"), false}, + {newMysqlErr(tmysql.ErrUnknown, "can't drop column with index"), false}, + {newMysqlErr(tmysql.ErrUnknown, "Information schema is out of date"), true}, + {newMysqlErr(tmysql.ErrUnknown, "Information schema is changed"), true}, } for _, t := range cases { diff --git a/tests/load_interrupt/run.sh b/tests/load_interrupt/run.sh index c188bca7dd..5d10e7bd75 100755 --- a/tests/load_interrupt/run.sh +++ b/tests/load_interrupt/run.sh @@ -60,7 +60,7 @@ function run() { check_row_count 2 # only failed at the first two time, will retry later and success - export GO_FAILPOINTS='github.com/pingcap/dm/loader/LoadExecCreateTableFailed=3*return("1105")' + export GO_FAILPOINTS='github.com/pingcap/dm/loader/LoadExecCreateTableFailed=3*return("1213")' # ER_LOCK_DEADLOCK, retryable error code run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT