*: added support for storing checkpoint at local filesystem (#90)

* *: added support for storing checkpoint at local filesystem

* config, tests: change default checkpoint driver to "file"

* mydump: regenerate the parser to avoid golang/go#27350 during coverage

* checkpoints: addressed comments

Makefile

@@ -48,8 +48,9 @@ checksuccess:
 
 data_parsers:
 	ragel -Z -G2 -o tmp_parser.go lightning/mydump/parser.rl
-	@echo '// Code generated by ragel DO NOT EDIT.' | cat - tmp_parser.go > lightning/mydump/parser_generated.go
+	@echo '// Code generated by ragel DO NOT EDIT.' | cat - tmp_parser.go | sed 's|//line |//.... |g' > lightning/mydump/parser_generated.go
 	@rm tmp_parser.go
+	PATH="$(GOPATH)/bin":$(PATH) protoc -I. -I"$(GOPATH)/src" lightning/restore/file_checkpoints.proto --gogofaster_out=.
 
 lightning:
 	$(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS)' -o $(LIGHTNING_BIN) cmd/main.go

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/cznic/strutil v0.0.0-20150430124730-1eb03e3cc9d3
 	github.com/cznic/y v0.0.0-20160420101755-9fdf92d4aac0
 	github.com/go-sql-driver/mysql v1.4.0
+	github.com/gogo/protobuf v1.1.1
 	github.com/joho/sqltocsv v0.0.0-20180904231936-b24deec2b806
 	github.com/pingcap/check v0.0.0-20171206051426-1c287c953996
 	github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26

lightning/config/config.go

@@ -97,6 +97,7 @@ type Checkpoint struct {
 	Enable           bool   `toml:"enable" json:"enable"`
 	Schema           string `toml:"schema" json:"schema"`
 	DSN              string `toml:"dsn" json:"-"` // DSN may contain password, don't expose this to JSON.
+	Driver           string `toml:"driver" json:"driver"`
 	KeepAfterSuccess bool   `toml:"keep-after-success" json:"keep-after-success"`
 }
 
@@ -175,8 +176,16 @@ func (cfg *Config) Load() error {
 	if len(cfg.Checkpoint.Schema) == 0 {
 		cfg.Checkpoint.Schema = "tidb_lightning_checkpoint"
 	}
+	if len(cfg.Checkpoint.Driver) == 0 {
+		cfg.Checkpoint.Driver = "file"
+	}
 	if len(cfg.Checkpoint.DSN) == 0 {
-		cfg.Checkpoint.DSN = common.ToDSN(cfg.TiDB.Host, cfg.TiDB.Port, cfg.TiDB.User, cfg.TiDB.Psw)
+		switch cfg.Checkpoint.Driver {
+		case "mysql":
+			cfg.Checkpoint.DSN = common.ToDSN(cfg.TiDB.Host, cfg.TiDB.Port, cfg.TiDB.User, cfg.TiDB.Psw)
+		case "file":
+			cfg.Checkpoint.DSN = "/tmp/" + cfg.Checkpoint.Schema + ".pb"
+		}
 	}
 
 	return nil