Skip to content

Commit

Permalink
dm: load table structure from dump files (pingcap#3295) (pingcap#4163)
Browse files Browse the repository at this point in the history
  • Loading branch information
ti-chi-bot authored and 3AceShowHand committed Jan 13, 2022
1 parent fcbf280 commit 76de75d
Show file tree
Hide file tree
Showing 4 changed files with 10 additions and 14 deletions.
1 change: 0 additions & 1 deletion dm/loader/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,6 @@ func cleanDumpFiles(cfg *config.SubTaskConfig) {
var lastErr error
for f := range files {
if strings.HasSuffix(f, ".sql") {
// TODO: table structure files are not used now, but we plan to used them in future so not delete them
if strings.HasSuffix(f, "-schema-create.sql") || strings.HasSuffix(f, "-schema.sql") {
continue
}
Expand Down
7 changes: 5 additions & 2 deletions dm/syncer/syncer.go
Original file line number Diff line number Diff line change
Expand Up @@ -1351,7 +1351,11 @@ func (s *Syncer) Run(ctx context.Context) (err error) {
if s.cfg.Mode == config.ModeAll && fresh {
delLoadTask = true
flushCheckpoint = true
// TODO: loadTableStructureFromDump in future
err = s.loadTableStructureFromDump(ctx)
if err != nil {
tctx.L().Warn("error happened when load table structure from dump files", zap.Error(err))
cleanDumpFile = false
}
} else {
cleanDumpFile = false
}
Expand Down Expand Up @@ -2766,7 +2770,6 @@ func (s *Syncer) genRouter() error {
return nil
}

//nolint:unused
func (s *Syncer) loadTableStructureFromDump(ctx context.Context) error {
logger := s.tctx.L()

Expand Down
11 changes: 1 addition & 10 deletions dm/tests/downstream_more_column/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,8 @@ function run() {
run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1<100;" "count(1): 2"

run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
"query-status test" \
"sourceTable: \`${db1}\`.\`${tb1}\`" 1 \
"targetTable: \`${db}\`.\`${tb}\`" 1 \
"Column count doesn't match value count" 1
run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
"operate-schema set -s mysql-replica-01 test -d ${db1} -t ${tb1} $cur/data/schema.sql" \
"\"result\": true" 2
# we load table structure from dump files, so there's no error now

run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
"resume-task test"
# check incremental data
run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1>100 and c1<1000;" "count(1): 2"

Expand Down
5 changes: 4 additions & 1 deletion dm/tests/shardddl1/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ function DM_001_CASE() {
# schema tracker could track per table without error
run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
"query-status test" \
"Duplicate column name 'new_col1'" 1
"\"result\": true" 2 \
"\"synced\": true" 1
# only downstream sees a duplicate error, but currently ignored by DM
check_log_contain_with_retry "Duplicate column name 'new_col1'" $WORK_DIR/worker1/log/dm-worker.log $WORK_DIR/worker2/log/dm-worker.log
}

function DM_001() {
Expand Down

0 comments on commit 76de75d

Please sign in to comment.