fix binary value null value problem (pingcap/dumpling#137)

* fix binary value null value problem * fix csv * address comments * address comments
tisonkun · Oct 20, 2021 · 9a9e38d · 9a9e38d
1 parent f89900c
commit 9a9e38d
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 11 deletions.
diff --git a/dumpling/go.sum b/dumpling/go.sum
@@ -257,8 +257,6 @@ github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200730093003-dc8c75cf7ca0 h1:cSHKKU5Tt4
 github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200730093003-dc8c75cf7ca0/go.mod h1:szYFB2rf8yrSGJuI8hm9RLWvsK+xt1exLTj511WPCnE=
 github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
-github.com/pingcap/tidb-tools v4.0.5-0.20200805025317-02a16e0521cb+incompatible h1:obFDQU7XvnCj8CiaXFP+bWaiQDp15Ueynk8cC0s5Utk=
-github.com/pingcap/tidb-tools v4.0.5-0.20200805025317-02a16e0521cb+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tidb-tools v4.0.5-0.20200817064459-ba61a7376547+incompatible h1:KGkKSxJ4t5F3Ys0ox144M5vGWm8NuHkXFyETFtakKW4=
 github.com/pingcap/tidb-tools v4.0.5-0.20200817064459-ba61a7376547+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

diff --git a/dumpling/install.sh b/dumpling/install.sh
@@ -2,9 +2,7 @@
 
 set -e
 
-# FIXME: change to latest version after lightning fix issue
-# https://github.com/pingcap/tidb-lightning/issues/277
-TAG="v3.1.0-beta.1"
+TAG="nightly"
 pwd=$(pwd)
 
 mkdir bin/
@@ -14,8 +12,9 @@ wget http://download.pingcap.org/tidb-toolkit-$TAG-linux-amd64.tar.gz -O tools.t
 tar -xzvf tools.tar.gz
 mv tidb-toolkit-$TAG-linux-amd64/bin/* bin/
 
+TIDB_TAG="v4.0.4"
 # download tidb-server
-git clone -b $TAG https://github.com/pingcap/tidb
+git clone -b $TIDB_TAG https://github.com/pingcap/tidb
 cd $pwd/tidb && make
 cd $pwd
 mv tidb/bin/tidb-server bin/
diff --git a/dumpling/tests/e2e/run.sh b/dumpling/tests/e2e/run.sh
@@ -16,10 +16,13 @@ run_sql "drop database if exists $DB_NAME;"
 
 # build data on mysql
 run_sql "create database $DB_NAME;"
-run_sql "create table $DB_NAME.$TABLE_NAME (a int(255));"
+run_sql "create table $DB_NAME.$TABLE_NAME (a int(255), b blob);"
 
 # insert 100 records
-run_sql "insert into $DB_NAME.$TABLE_NAME values $(seq -s, 100 | sed 's/,*$//g' | sed "s/[0-9]*/('1')/g");"
+run_sql "insert into $DB_NAME.$TABLE_NAME (a) values $(seq -s, 100 | sed 's/,*$//g' | sed "s/[0-9]*/('1')/g");"
+
+# insert blob records
+run_sql "insert into $DB_NAME.$TABLE_NAME (b) values (x''),(null),('0'),('1');"
 
 # dumping
 export DUMPLING_TEST_DATABASE=$DB_NAME

diff --git a/dumpling/v4/export/sql_type.go b/dumpling/v4/export/sql_type.go
@@ -228,13 +228,17 @@ func (s *SQLTypeBytes) ReportSize() uint64 {
 }
 
 func (s *SQLTypeBytes) WriteToBuffer(bf *bytes.Buffer, _ bool) {
-	fmt.Fprintf(bf, "x'%x'", s.RawBytes)
+	if s.RawBytes != nil {
+		fmt.Fprintf(bf, "x'%x'", s.RawBytes)
+	} else {
+		bf.WriteString(nullValue)
+	}
 }
 
-func (s *SQLTypeBytes) WriteToBufferInCsv(bf *bytes.Buffer, _ bool, opt *csvOption) {
+func (s *SQLTypeBytes) WriteToBufferInCsv(bf *bytes.Buffer, escapeBackslash bool, opt *csvOption) {
 	if s.RawBytes != nil {
 		bf.Write(opt.delimiter)
-		bf.Write(s.RawBytes)
+		escape(s.RawBytes, bf, getEscapeQuotation(escapeBackslash, opt.delimiter))
 		bf.Write(opt.delimiter)
 	} else {
 		bf.WriteString(opt.nullValue)