glue: create schema/table directly with info

go.mod

@@ -21,11 +21,11 @@ require (
 	github.com/onsi/gomega v1.8.1 // indirect
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
 	github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
-	github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75
+	github.com/pingcap/kvproto v0.0.0-20200330093347-98f910b71904
 	github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
-	github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87
+	github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5
 	github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3
-	github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897
+	github.com/pingcap/tidb v0.0.0-20200401141416-959eca8f3a39
 	github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible
 	github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60
 	github.com/prometheus/client_golang v1.0.0
@@ -37,7 +37,7 @@ require (
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
 	go.opencensus.io v0.22.2 // indirect
-	go.uber.org/zap v1.14.0
+	go.uber.org/zap v1.14.1
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	google.golang.org/api v0.14.0
 	google.golang.org/grpc v1.25.1

go.sum

@@ -363,24 +363,23 @@ github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17Xtb
 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
 github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75 h1:DB3NTM0ilba/6sW+vccdEnP10bVvrVunDwWvRa0hSKc=
-github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
+github.com/pingcap/kvproto v0.0.0-20200330093347-98f910b71904 h1:pMFUXvhJ62hX8m0Q4RsL7L+hSW1mAMG26So5eFMoAtI=
+github.com/pingcap/kvproto v0.0.0-20200330093347-98f910b71904/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87 h1:533jEUp3mtfWjk0el+awLbyGVxiHcUIGWcR1Y7gB+fg=
-github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4=
+github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5 h1:fXVqoeYfV+xI8K2he5NNv00c6YksrjeM6+vkNo1ZK2Q=
+github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4=
 github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk=
 github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3/go.mod h1:25GfNw6+Jcr9kca5rtmTb4gKCJ4jOpow2zV2S9Dgafs=
 github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1 h1:YUnUZ914SHFMsOSe/xgH5DKK/thtRma8X8hcszRo3CA=
 github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb h1:bDbgLaNTRNK6Qw7KjvEqqfCQstY8WMEcXyXTU7yzYKg=
 github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
-github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897 h1:wTNFJMM6GNmG09YoN3/3K8BqiK74zKkurjm4iY+m2mI=
-github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897/go.mod h1:4CGOiKZSaOU/Da3QYMtp0c3uBE2SxpcLOpESXmeQhcs=
+github.com/pingcap/tidb v0.0.0-20200401141416-959eca8f3a39 h1:nYRL69Qc4kuvp+tlDNB5wXjvDetX0J7g0DsW4RQxfXM=
+github.com/pingcap/tidb v0.0.0-20200401141416-959eca8f3a39/go.mod h1:btnHsqUQvJnY18+OP2Z6MCRq1tX4B8JUCrmqctSKxOg=
 github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible h1:84F7MFMfdAYObrznvRslmVu43aoihrlL+7mMyMlOi0o=
 github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
@@ -535,8 +534,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.12.0 h1:dySoUQPFBGj6xwjmBzageVL8jGi8uxc6bEmJQjA06bw=
 go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
-go.uber.org/zap v1.14.0 h1:/pduUoebOeeJzTDFuoMgC6nRkiasr1sBCIEorly7m4o=
-go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo=
+go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -676,8 +675,8 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200301222351-066e0c02454c h1:FD7jysxM+EJqg5UYYy3XYDsAiUickFsn4UiaanJkf8c=
 golang.org/x/tools v0.0.0-20200301222351-066e0c02454c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200313205530-4303120df7d8 h1:gkI/wGGwpcG5W4hLCzZNGxA4wzWBGGDStRI1MrjDl2Q=
-golang.org/x/tools v0.0.0-20200313205530-4303120df7d8/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1 h1:odiryKYJy7CjdrZxhrcE1Z8L9+kGyGZOnfpuauvdCeU=
+golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/backup/client.go

@@ -224,6 +224,16 @@ func BuildBackupRangeAndSchema(
 				zap.Stringer("table", tableInfo.Name),
 				zap.Int64("AutoIncID", globalAutoID))
 
+			// remove all non-public indices
+			n := 0
+			for _, index := range tableInfo.Indices {
+				if index.State == model.StatePublic {
+					tableInfo.Indices[n] = index
+					n++
+				}
+			}
+			tableInfo.Indices = tableInfo.Indices[:n]
+
 			if dbData == nil {
 				dbData, err = json.Marshal(dbInfo)
 				if err != nil {

pkg/glue/glue.go

@@ -9,7 +9,6 @@ import (
 	pd "github.com/pingcap/pd/v4/client"
 	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/kv"
-	"github.com/pingcap/tidb/meta/autoid"
 )
 
 // Glue is an abstraction of TiDB function calls used in BR.
@@ -28,8 +27,8 @@ type Glue interface {
 // Session is an abstraction of the session.Session interface.
 type Session interface {
 	Execute(ctx context.Context, sql string) error
-	ShowCreateDatabase(schema *model.DBInfo) (string, error)
-	ShowCreateTable(table *model.TableInfo, allocator autoid.Allocator) (string, error)
+	CreateDatabase(ctx context.Context, schema *model.DBInfo) error
+	CreateTable(ctx context.Context, dbName model.CIStr, table *model.TableInfo) error
 	Close()
 }
 

pkg/gluetidb/glue.go

@@ -3,15 +3,14 @@
 package gluetidb
 
 import (
-	"bytes"
 	"context"
 
 	"github.com/pingcap/parser/model"
+	"github.com/pingcap/parser/mysql"
 	pd "github.com/pingcap/pd/v4/client"
+	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/domain"
-	"github.com/pingcap/tidb/executor"
 	"github.com/pingcap/tidb/kv"
-	"github.com/pingcap/tidb/meta/autoid"
 	"github.com/pingcap/tidb/session"
 
 	"github.com/pingcap/br/pkg/glue"
@@ -62,22 +61,29 @@ func (gs *tidbSession) Execute(ctx context.Context, sql string) error {
 	return err
 }
 
-// ShowCreateDatabase implements glue.Session
-func (gs *tidbSession) ShowCreateDatabase(schema *model.DBInfo) (string, error) {
-	var buf bytes.Buffer
-	if err := executor.ConstructResultOfShowCreateDatabase(gs.se, schema, true, &buf); err != nil {
-		return "", err
+// CreateDatabase implements glue.Session
+func (gs *tidbSession) CreateDatabase(ctx context.Context, schema *model.DBInfo) error {
+	d := domain.GetDomain(gs.se).DDL()
+	schema = schema.Clone()
+	if len(schema.Charset) == 0 {
+		schema.Charset = mysql.DefaultCharset
 	}
-	return buf.String(), nil
+	return d.CreateSchemaWithInfo(gs.se, schema, ddl.OnExistIgnore, true)
 }
 
-// ShowCreateTable implements glue.Session
-func (gs *tidbSession) ShowCreateTable(table *model.TableInfo, allocator autoid.Allocator) (string, error) {
-	var buf bytes.Buffer
-	if err := executor.ConstructResultOfShowCreateTable(gs.se, table, allocator, &buf); err != nil {
-		return "", err
+// CreateTable implements glue.Session
+func (gs *tidbSession) CreateTable(ctx context.Context, dbName model.CIStr, table *model.TableInfo) error {
+	d := domain.GetDomain(gs.se).DDL()
+
+	// Clone() does not clone partitions yet :(
+	table = table.Clone()
+	if table.Partition != nil {
+		newPartition := *table.Partition
+		newPartition.Definitions = append([]model.PartitionDefinition{}, table.Partition.Definitions...)
+		table.Partition = &newPartition
 	}
-	return buf.String(), nil
+
+	return d.CreateTableWithInfo(gs.se, dbName, table, ddl.OnExistIgnore, true)
 }
 
 // Close implements glue.Session

pkg/restore/db.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"fmt"
 	"sort"
-	"strings"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/log"
@@ -70,57 +69,28 @@ func (db *DB) ExecDDL(ctx context.Context, ddlJob *model.Job) error {
 
 // CreateDatabase executes a CREATE DATABASE SQL.
 func (db *DB) CreateDatabase(ctx context.Context, schema *model.DBInfo) error {
-	createSQL, err := db.se.ShowCreateDatabase(schema)
+	err := db.se.CreateDatabase(ctx, schema)
 	if err != nil {
-		log.Error("build create database SQL failed", zap.Stringer("db", schema.Name), zap.Error(err))
-		return errors.Trace(err)
-	}
-	err = db.se.Execute(ctx, createSQL)
-	if err != nil {
-		log.Error("create database failed", zap.String("query", createSQL), zap.Error(err))
+		log.Error("create database failed", zap.Stringer("db", schema.Name), zap.Error(err))
 	}
 	return errors.Trace(err)
 }
 
 // CreateTable executes a CREATE TABLE SQL.
 func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error {
-	tableInfo := table.Info
-	createSQL, err := db.se.ShowCreateTable(tableInfo, newIDAllocator(tableInfo.AutoIncID))
-	if err != nil {
-		log.Error(
-			"build create table SQL failed",
-			zap.Stringer("db", table.Db.Name),
-			zap.Stringer("table", tableInfo.Name),
-			zap.Error(err))
-		return errors.Trace(err)
-	}
-	switchDbSQL := fmt.Sprintf("use %s;", utils.EncloseName(table.Db.Name.O))
-	err = db.se.Execute(ctx, switchDbSQL)
-	if err != nil {
-		log.Error("switch db failed",
-			zap.String("SQL", switchDbSQL),
-			zap.Stringer("db", table.Db.Name),
-			zap.Error(err))
-		return errors.Trace(err)
-	}
-	// Insert `IF NOT EXISTS` statement to skip the created tables
-	words := strings.SplitN(createSQL, " ", 3)
-	if len(words) > 2 && strings.ToUpper(words[0]) == "CREATE" && strings.ToUpper(words[1]) == "TABLE" {
-		createSQL = "CREATE TABLE IF NOT EXISTS " + words[2]
-	}
-	err = db.se.Execute(ctx, createSQL)
+	err := db.se.CreateTable(ctx, table.Db.Name, table.Info)
 	if err != nil {
 		log.Error("create table failed",
-			zap.String("SQL", createSQL),
 			zap.Stringer("db", table.Db.Name),
 			zap.Stringer("table", table.Info.Name),
 			zap.Error(err))
 		return errors.Trace(err)
 	}
 	alterAutoIncIDSQL := fmt.Sprintf(
-		"alter table %s auto_increment = %d",
-		utils.EncloseName(tableInfo.Name.O),
-		tableInfo.AutoIncID)
+		"alter table %s.%s auto_increment = %d",
+		utils.EncloseName(table.Db.Name.O),
+		utils.EncloseName(table.Info.Name.O),
+		table.Info.AutoIncID)
 	err = db.se.Execute(ctx, alterAutoIncIDSQL)
 	if err != nil {
 		log.Error("alter AutoIncID failed",

pkg/restore/util.go

@@ -16,7 +16,6 @@ import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/log"
 	"github.com/pingcap/parser/model"
-	"github.com/pingcap/tidb/meta/autoid"
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/util/codec"
 	"go.uber.org/zap"
@@ -28,44 +27,6 @@ import (
 
 var recordPrefixSep = []byte("_r")
 
-// idAllocator always returns a specified ID
-type idAllocator struct {
-	id int64
-}
-
-func newIDAllocator(id int64) *idAllocator {
-	return &idAllocator{id: id}
-}
-
-func (alloc *idAllocator) Alloc(tableID int64, n uint64, increment, offset int64) (min int64, max int64, err error) {
-	return alloc.id, alloc.id, nil
-}
-
-func (alloc *idAllocator) AllocSeqCache(sequenceID int64) (min int64, max int64, round int64, err error) {
-	// TODO fix this function after support backup sequence
-	return 0, 0, 0, nil
-}
-
-func (alloc *idAllocator) Rebase(tableID, newBase int64, allocIDs bool) error {
-	return nil
-}
-
-func (alloc *idAllocator) Base() int64 {
-	return alloc.id
-}
-
-func (alloc *idAllocator) End() int64 {
-	return alloc.id
-}
-
-func (alloc *idAllocator) NextGlobalAutoID(tableID int64) (int64, error) {
-	return alloc.id, nil
-}
-
-func (alloc *idAllocator) GetType() autoid.AllocatorType {
-	return autoid.RowIDAllocType
-}
-
 // GetRewriteRules returns the rewrite rule of the new table and the old table.
 func GetRewriteRules(
 	newTable *model.TableInfo,

tests/_utils/run_services

@@ -38,6 +38,9 @@ stop_services() {
 start_services() {
     stop_services
 
+    TIDB_CONFIG="${1-tests}/config/tidb.toml"
+    TIKV_CONFIG="${1-tests}/config/tikv.toml"
+
     echo "Starting PD..."
     mkdir -p "$TEST_DIR/pd"
     bin/pd-server \
@@ -63,7 +66,7 @@ start_services() {
             -A "$TIKV_ADDR$i" \
             --status-addr "$TIKV_STATUS_ADDR$i" \
             --log-file "$TEST_DIR/tikv${i}.log" \
-            -C "tests/config/tikv.toml" \
+            -C "$TIKV_CONFIG" \
             -s "$TEST_DIR/tikv${i}" &
     done
 
@@ -83,7 +86,7 @@ start_services() {
         --status 10080 \
         --store tikv \
         --path "$PD_ADDR" \
-        --config "tests/config/tidb.toml" \
+        --config "$TIDB_CONFIG" \
         --log-file "$TEST_DIR/tidb.log" &
 
     echo "Verifying TiDB is started..."

tests/br_alter_pk_server/config/tidb.toml

@@ -0,0 +1,8 @@
+# config of tidb
+
+# Schema lease duration
+# There are lot of ddl in the tests, setting this
+# to 360s to test whther BR is gracefully shutdown.
+lease = "360s"
+
+alter-primary-key = true

tests/br_alter_pk_server/config/tikv.toml

@@ -0,0 +1,14 @@
+# config of tikv
+
+[coprocessor]
+region-max-keys = 20
+region-split-keys = 12
+
+[rocksdb]
+max-open-files = 4096
+[raftdb]
+max-open-files = 4096
+[raftstore]
+# true (default value) for high reliability, this can prevent data loss when power failure.
+sync-log = false
+capacity = "10GB"

tests/br_alter_pk_server/run.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source $cur/../_utils/run_services
+
+DB="$TEST_NAME"
+
+# prepare database
+echo "Restart cluster with alter-primary-key = true"
+start_services "$cur"
+
+run_sql "drop schema if exists $DB;"
+run_sql "create schema $DB;"
+
+run_sql "create table $DB.a (a int primary key, b int unique);"
+run_sql "insert into $DB.a values (42, 42);"
+
+# backup
+run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB"
+
+# restore
+run_sql "drop schema $DB;"
+run_br --pd $PD_ADDR restore db --db "$DB" -s "local://$TEST_DIR/$DB"
+
+run_sql "drop schema $DB;"
+echo "Restart service with alter-primary-key = false"
+start_services