pingcap · GMHDBJD · Jul 8, 2020 · Jul 7, 2020 · Jul 7, 2020 · Jul 8, 2020
diff --git a/_utils/terror_gen/errors_release.txt b/_utils/terror_gen/errors_release.txt
@@ -451,4 +451,5 @@ ErrSchedulerSubTaskExist,[code=46014:class=scheduler:scope=internal:level=medium
 ErrSchedulerSubTaskStageInvalidUpdate,[code=46015:class=dm-master:scope=internal:level=medium], "Message: invalid new expectant subtask stage %s"
 ErrSchedulerSubTaskOpTaskNotExist,[code=46016:class=dm-master:scope=internal:level=medium], "Message: subtasks with name %s need to be operate not exist"
 ErrSchedulerSubTaskOpSourceNotExist,[code=46017:class=dm-master:scope=internal:level=medium], "Message: sources %v need to be operate not exist"
-ErrNotSet,[code=48001:class=not-set:scope=not-set:level=high]
+ErrCtlGRPCCreateConn,[code=48001:class=dmctl:scope=internal:level=high], "Message: can not create grpc connection, Workaround: Please check your network connection."
+ErrNotSet,[code=50000:class=not-set:scope=not-set:level=high]
diff --git a/cmd/dm-ctl/main.go b/cmd/dm-ctl/main.go
@@ -28,6 +28,7 @@ import (
 	"github.com/pingcap/dm/dm/ctl"
 	"github.com/pingcap/dm/dm/ctl/common"
 	"github.com/pingcap/dm/pkg/log"
+	"github.com/pingcap/dm/pkg/terror"
 	"github.com/pingcap/dm/pkg/utils"
 )
 
@@ -148,7 +149,7 @@ func main() {
 
 	err = ctl.Init(cfg)
 	if err != nil {
-		fmt.Printf("init control error %v", errors.ErrorStack(err))
+		common.PrintLines("%v", terror.Message(err))
 		os.Exit(2)
 	}
 	if lenCmdArgs > 0 {

diff --git a/dm/ctl/common/util.go b/dm/ctl/common/util.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/pingcap/dm/dm/pb"
 	parserpkg "github.com/pingcap/dm/pkg/parser"
+	"github.com/pingcap/dm/pkg/terror"
 
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"
@@ -45,9 +46,9 @@ func InitUtils(cfg *Config) error {
 
 // InitClient initializes dm-master client
 func InitClient(addr string) error {
-	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBackoffMaxDelay(3*time.Second))
+	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBackoffMaxDelay(3*time.Second), grpc.WithBlock(), grpc.WithTimeout(3*time.Second))
 	if err != nil {
-		return errors.Trace(err)
+		return terror.ErrCtlGRPCCreateConn.AnnotateDelegate(err, "can't connect to %s", addr)
 	}
 	masterClient = pb.NewMasterClient(conn)
 	return nil

diff --git a/errors.toml b/errors.toml
@@ -2716,7 +2716,13 @@ description = ""
 workaround = ""
 tags = ["internal", "medium"]
 
-[error.DM-not-set-48001]
+[error.DM-dmctl-48001]
+message = "can not create grpc connection"
+description = ""
+workaround = "Please check your network connection."
+tags = ["internal", "high"]
+
+[error.DM-not-set-50000]
 message = ""
 description = ""
 workaround = ""

diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go
@@ -546,8 +546,14 @@ const (
 	codeSchedulerSubTaskOpSourceNotExist
 )
 
+// dmctl error code
 const (
-	codeNotSet ErrCode = iota + 48001
+	codeCtlGRPCCreateConn ErrCode = iota + 48001
+)
+
+// default error code
+const (
+	codeNotSet ErrCode = iota + 50000
 )
 
 // Error instances
@@ -1062,6 +1068,9 @@ var (
 	ErrSchedulerSubTaskOpTaskNotExist     = New(codeSchedulerSubTaskOpTaskNotExist, ClassDMMaster, ScopeInternal, LevelMedium, "subtasks with name %s need to be operate not exist", "")
 	ErrSchedulerSubTaskOpSourceNotExist   = New(codeSchedulerSubTaskOpSourceNotExist, ClassDMMaster, ScopeInternal, LevelMedium, "sources %v need to be operate not exist", "")
 
-	// default error code
+	// dmctl
+	ErrCtlGRPCCreateConn = New(codeCtlGRPCCreateConn, ClassDMCtl, ScopeInternal, LevelHigh, "can not create grpc connection", "Please check your network connection.")
+
+	// default error
 	ErrNotSet = New(codeNotSet, ClassNotSet, ScopeNotSet, LevelHigh, "", "")
 )
diff --git a/pkg/terror/terror.go b/pkg/terror/terror.go
@@ -48,6 +48,7 @@ const (
 	ClassDMTracer
 	ClassSchemaTracker
 	ClassScheduler
+	ClassDMCtl
 	ClassNotSet
 )
 
@@ -68,6 +69,7 @@ var errClass2Str = map[ErrClass]string{
 	ClassDMTracer:      "dm-tracer",
 	ClassSchemaTracker: "schema-tracker",
 	ClassScheduler:     "scheduler",
+	ClassDMCtl:         "dmctl",
 	ClassNotSet:        "not-set",
 }
 

diff --git a/tests/dmctl_advance/check_list/migrate_relay.sh b/tests/dmctl_advance/check_list/migrate_relay.sh
@@ -12,9 +12,3 @@ function migrate_relay_without_worker() {
         "migrate-relay $SOURCE_ID1 bin-000001 $binlog_pos" \
         "strconv.Atoi: parsing \"$binlog_pos\": invalid syntax" 1
 }
-
-function migrate_relay_while_master_down() {
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "migrate-relay $SOURCE_ID1 bin-000001 194" \
-        "can not migrate relay" 1
-}
diff --git a/tests/dmctl_advance/check_list/query_error.sh b/tests/dmctl_advance/check_list/query_error.sh
@@ -6,8 +6,3 @@ function query_error_wrong_arg() {
         "query-error \[-s source ...\] \[task-name\]" 1
 }
 
-function query_error_while_master_down() {
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "query-error -s $SOURCE_ID1 test-task" \
-        "query error failed" 1
-}
diff --git a/tests/dmctl_advance/check_list/sql_replace.sh b/tests/dmctl_advance/check_list/sql_replace.sh
@@ -18,9 +18,3 @@ function sql_replace_non_sharding_without_one_worker() {
         "sql-replace test-task --source $SOURCE_ID1,$SOURCE_ID2 --binlog-pos mysql-bin:13426 sql1" \
         "should only specify one source, but got \[$SOURCE_ID1 $SOURCE_ID2\]" 1
 }
-
-function sql_replace_while_master_down() {
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "sql-replace test-task --sharding --sql-pattern ~(?i)ALTER\\s+TABLE\\s+ ALTER TABLE tbl DROP COLUMN col" \
-        "can not replace SQL:" 1
-}
diff --git a/tests/dmctl_advance/check_list/sql_skip.sh b/tests/dmctl_advance/check_list/sql_skip.sh
@@ -37,9 +37,3 @@ function sql_skip_non_sharding_without_one_worker() {
         "sql-skip test-task --source $SOURCE_ID1,$SOURCE_ID2 --binlog-pos mysql-bin:13426" \
         "should only specify one source, but got \[$SOURCE_ID1 $SOURCE_ID2\]" 1
 }
-
-function sql_skip_while_master_down() {
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "sql-skip test-task --sharding --sql-pattern ~(?i)ALTER\\s+TABLE\\s+" \
-        "can not skip SQL:" 1
-}
diff --git a/tests/dmctl_advance/check_list/switch_relay_master.sh b/tests/dmctl_advance/check_list/switch_relay_master.sh
@@ -11,9 +11,3 @@ function switch_relay_master_without_worker() {
         "switch-relay-master" \
         "must specify at least one source (\`-s\` \/ \`--source\`)" 1
 }
-
-function switch_relay_master_while_master_down() {
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "switch-relay-master -s $SOURCE_ID1" \
-        "can not switch relay's master server (in sources \[$SOURCE_ID1\]):" 1
-}
diff --git a/tests/dmctl_advance/check_list/unlock_ddl_lock.sh b/tests/dmctl_advance/check_list/unlock_ddl_lock.sh
@@ -12,9 +12,3 @@ function unlock_ddl_lock_invalid_force_remove() {
         "unlock-ddl-lock lock_id --force-remove=$force_remove_val" \
         "Error: invalid argument \"$force_remove_val\" for \"-f, --force-remove\"" 1
 }
-function unlock_ddl_lock_while_master_down() {
-    lock_id="test-\`shard_db\`.\`shard_table\`"
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "unlock-ddl-lock $lock_id" \
-        "can not unlock DDL lock $lock_id" 1
-}
diff --git a/tests/dmctl_advance/conf/dm-master.toml b/tests/dmctl_advance/conf/dm-master.toml
@@ -0,0 +1,4 @@
+# Master Configuration.
+
+master-addr = ":8261"
+advertise-addr = "127.0.0.1:8261"
diff --git a/tests/dmctl_advance/run.sh b/tests/dmctl_advance/run.sh
@@ -14,33 +14,26 @@ SQL_RESULT_FILE="$TEST_DIR/sql_res.$TEST_NAME.txt"
 function usage_and_arg_test() {
     migrate_relay_wrong_arg
     migrate_relay_without_worker
-    # todo: no sense to fix
-    #migrate_relay_while_master_down
 
     switch_relay_master_wrong_arg
     switch_relay_master_without_worker
-    switch_relay_master_while_master_down
 
     unlock_ddl_lock_wrong_arg
     unlock_ddl_lock_invalid_force_remove
-    unlock_ddl_lock_while_master_down
 
     query_error_wrong_arg
-    query_error_while_master_down
 
     sql_skip_wrong_arg
     sql_skip_binlogpos_sqlpattern_conflict
     sql_skip_invalid_binlog_pos
     sql_skip_invalid_regex
     sql_skip_sharding_with_binlogpos
     sql_skip_non_sharding_without_one_worker
-    sql_skip_while_master_down
 
     sql_replace_wrong_arg
     sql_replace_invalid_binlog_pos
     sql_replace_non_sharding_without_one_worker
     # TODO: check SQLs error test
-    sql_replace_while_master_down
 }
 
 function run() {
@@ -50,6 +43,8 @@ function run() {
     done
     cd -
 
+    run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
+    check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
     usage_and_arg_test
 }
 

diff --git a/tests/dmctl_basic/check_list/check_task.sh b/tests/dmctl_basic/check_list/check_task.sh
@@ -12,14 +12,6 @@ function check_task_wrong_config_file() {
         "get file content error" 1
 }
 
-# run this check if DM-master is not available
-function check_task_while_master_down() {
-    task_conf=$1
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "check-task $task_conf" \
-        "fail to check task" 1
-}
-
 function check_task_pass() {
     task_conf=$1
     run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \

diff --git a/tests/dmctl_basic/check_list/operate_source.sh b/tests/dmctl_basic/check_list/operate_source.sh
@@ -12,13 +12,6 @@ function operate_source_wrong_config_file() {
         "get file content error" 1
 }
 
-function operate_source_while_master_down() {
-    source_conf=$1
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "operate-source create $source_conf" \
-        "can not update task" 1
-}
-
 function operate_source_stop_not_created_config() {
     source_conf=$1
     run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \

diff --git a/tests/dmctl_basic/check_list/pause_relay.sh b/tests/dmctl_basic/check_list/pause_relay.sh
@@ -12,12 +12,6 @@ function pause_relay_wihout_worker() {
         "must specify at least one source" 1
 }
 
-function pause_relay_while_master_down() {
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "pause-relay -s $SOURCE_ID1 -s $SOURCE_ID2" \
-        "can not pause relay unit:" 1
-}
-
 function pause_relay_success() {
     run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
         "pause-relay -s $SOURCE_ID1 -s $SOURCE_ID2" \

diff --git a/tests/dmctl_basic/check_list/pause_task.sh b/tests/dmctl_basic/check_list/pause_task.sh
@@ -6,13 +6,6 @@ function pause_task_wrong_arg() {
         "pause-task \[-s source ...\] <task-name> \[flags\]" 1
 }
 
-function pause_task_while_master_down() {
-    task_name="any_task_name"
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "pause-task $task_name" \
-        "can not pause task $task_name:" 1
-}
-
 function pause_task_success() {
     task_name=$1
     run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \

diff --git a/tests/dmctl_basic/check_list/purge_relay.sh b/tests/dmctl_basic/check_list/purge_relay.sh
@@ -18,12 +18,6 @@ function purge_relay_filename_with_multi_workers() {
         "for --filename, can only specify one source per time" 1
 }
 
-function purge_relay_while_master_down() {
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "purge-relay --filename bin-000001 -s $SOURCE_ID1" \
-        "can not purge relay log files:" 1
-}
-
 function purge_relay_success() {
     binlog_file=$1
     source_id=$2

diff --git a/tests/dmctl_basic/check_list/query_status.sh b/tests/dmctl_basic/check_list/query_status.sh
@@ -9,7 +9,7 @@ function query_status_wrong_arg() {
 function query_status_wrong_params() {
     run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
         "query-status -s source-x task-y" \
-        "can not query task-y task's status(in sources \[source-x\]):" 1
+        "source-x relevant worker-client not found" 1
 }
 
 function query_status_with_no_tasks() {

diff --git a/tests/dmctl_basic/check_list/resume_task.sh b/tests/dmctl_basic/check_list/resume_task.sh
@@ -6,13 +6,6 @@ function resume_task_wrong_arg() {
         "resume-task \[-s source ...\] <task-name> \[flags\]" 1
 }
 
-function resume_task_while_master_down() {
-    task_name="any_task_name"
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "resume-task $task_name" \
-        "can not resume task $task_name:" 1
-}
-
 function resume_task_success() {
     task_name=$1
     run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \

diff --git a/tests/dmctl_basic/check_list/show_ddl_locks.sh b/tests/dmctl_basic/check_list/show_ddl_locks.sh
@@ -6,12 +6,6 @@ function show_ddl_locks_wrong_arg() {
         "show-ddl-locks \[-s source ...\] \[task-name\] \[flags\]" 1
 }
 
-function show_ddl_locks_while_master_down() {
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "show-ddl-locks task-name -s $SOURCE_ID1" \
-        "can not show DDL locks for task task-name and sources \[$SOURCE_ID1\]" 1
-}
-
 function show_ddl_locks_no_locks() {
     task_name=$1
     run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \

diff --git a/tests/dmctl_basic/check_list/start_task.sh b/tests/dmctl_basic/check_list/start_task.sh
@@ -11,10 +11,3 @@ function start_task_wrong_config_file() {
         "start-task not_exists_config_file" \
         "get file content error" 1
 }
-
-function start_task_while_master_down() {
-    task_conf=$1
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "start-task $task_conf" \
-        "can not start task:" 1
-}
diff --git a/tests/dmctl_basic/check_list/stop_task.sh b/tests/dmctl_basic/check_list/stop_task.sh
@@ -5,10 +5,3 @@ function stop_task_wrong_arg() {
         "stop-task" \
         "stop-task \[-s source ...\] <task-name> \[flags\]" 1
 }
-
-function stop_task_while_master_down() {
-    task_name="any_task_name"
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "stop-task $task_name" \
-        "can not stop task $task_name:" 1
-}
diff --git a/tests/dmctl_basic/check_list/update_master_config.sh b/tests/dmctl_basic/check_list/update_master_config.sh
@@ -12,13 +12,6 @@ function update_master_config_wrong_config_file() {
         "get file content error" 1
 }
 
-function update_master_config_while_master_down() {
-    master_conf=$1
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "update-master-config $master_conf" \
-        "can not update master config:" 1
-}
-
 function update_master_config_success() {
     master_conf=$1
     run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \

diff --git a/tests/dmctl_basic/check_list/update_relay.sh b/tests/dmctl_basic/check_list/update_relay.sh
@@ -19,13 +19,6 @@ function update_relay_should_specify_one_dm_worker() {
         "must specify one source (\`-s\` \/ \`--source\`)" 1
 }
 
-function update_relay_while_master_down() {
-    task_conf=$1
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "update-relay $task_conf -s $SOURCE_ID1" \
-        "can not update relay config:" 1
-}
-
 function update_relay_success() {
     task_conf=$1
     source_id=$2

diff --git a/tests/dmctl_basic/check_list/update_task.sh b/tests/dmctl_basic/check_list/update_task.sh
@@ -12,13 +12,6 @@ function update_task_wrong_config_file() {
         "get file content error" 1
 }
 
-function update_task_while_master_down() {
-    task_conf=$1
-    run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
-        "update-task $task_conf" \
-        "can not update task:" 1
-}
-
 function update_task_worker_not_found() {
     task_conf=$1
     not_found_source_id=$2