Release 3.1.0.beta2 (#162)

* restore: fix alter auto increment id for no-primary-key table (#139)

* restore: fix alter auto increment id for no-primary-key table

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* add integration test

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* *: use oracle package to manipulate ts and test gc safe point  (#121)

Signed-off-by: Neil Shen <overvenus@gmail.com>

* cmd: convert version command to flags (#144)

* cmd: convert version command to flags

Signed-off-by: Neil Shen <overvenus@gmail.com>

* address comments

Signed-off-by: Neil Shen <overvenus@gmail.com>

* *: add changelog and github templates (#143)

Signed-off-by: Neil Shen <overvenus@gmail.com>

* restore: merge tidb-tools/pkg/restore-util (#146)

* restore-util: Implement split/scatter (#274)

* implement split/scatter

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* init test

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* redesign output/input of the lib

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* update dependency

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* add commments and more tests

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* add ScanRegions interface to Client

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix potential data race

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* Apply suggestions from code review

Co-Authored-By: kennytm <kennytm@gmail.com>

* Update pkg/restore-util/client.go

Co-Authored-By: kennytm <kennytm@gmail.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* update dependency

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* resolve conflicts

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix prefix rewrite

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* add RewriteRule/skip failed scatter region/retry the SplitRegion

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix test

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* check if region has peer

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* more logs

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* restore-util: add split retry interval (#277)

* reset dependencies to release-3.1

* add split retry interval

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix go.sum

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* restore-util: wait for scatter region sequentially  (#279)

* wait for scatter region sequentially

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* restore-util: add on split hook (#281)

* restore-util: add on split hook

Signed-off-by: Neil Shen <overvenus@gmail.com>

* Nil check onSplit

Co-Authored-By: kennytm <kennytm@gmail.com>

* restore-util: fix returned new region is nil (#283)

* restore-util: fix returned new region is nil

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* more logs

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* *: gofmt

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* Apply suggestions from code review

Co-Authored-By: kennytm <kennytm@gmail.com>

* fix log

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* restore-util: call onSplit on splitByRewriteRules (#285)

Signed-off-by: Neil Shen <overvenus@gmail.com>

* restore-util: fix overlapped error message (#293)

* restore-util: fix overlapped error message

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix log message

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* reduce error trace

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix test

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* restore-util: log warning when cannot find matched rewrite rule (#299)

* restore-util: add method to set placement rules and store labels (#301)

* restore-util: add method to set placement rules and store labels

Signed-off-by: disksing <i@disksing.com>

* minor fix

Signed-off-by: disksing <i@disksing.com>

* address comment

Signed-off-by: disksing <i@disksing.com>

* add GetPlacementRules

Signed-off-by: disksing <i@disksing.com>

* fix test

Signed-off-by: disksing <i@disksing.com>

* restore-util: support batch split (#300)

* restore-util: support batch split

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* go fmt

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* Apply suggestions from code review

Co-Authored-By: kennytm <kennytm@gmail.com>

* address commits

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* Update pkg/restore-util/split.go

Co-Authored-By: kennytm <kennytm@gmail.com>

* add onSplit callback

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix test

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* restore-util: add upper bound time for waiting for scatter (#305)

* restore: fix scatter regions failed

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* add log

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* stop waiting for scatter after 3min

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* restore-util: fix wrong url (#306)

Signed-off-by: disksing <i@disksing.com>

* restore-util: add warning about unmatched table id (#313)

* restore-util: support table partition

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix log

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* warn table id does not match

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* add unit tests

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* Apply suggestions from code review

Co-Authored-By: Neil Shen <overvenus@gmail.com>

* fix compile error

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix test

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

Co-authored-by: Ian <ArGregoryIan@gmail.com>
Co-authored-by: Neil Shen <overvenus@gmail.com>

* *: prune tidb-tools

Signed-off-by: Neil Shen <overvenus@gmail.com>

* restore: address linters suggestions

Signed-off-by: Neil Shen <overvenus@gmail.com>

* restore: merge restoreutil into restore

Signed-off-by: Neil Shen <overvenus@gmail.com>

* address comment

Signed-off-by: Neil Shen <overvenus@gmail.com>

Co-authored-by: 5kbpers <20279863+5kbpers@users.noreply.github.com>
Co-authored-by: kennytm <kennytm@gmail.com>
Co-authored-by: disksing <i@disksing.com>
Co-authored-by: Ian <ArGregoryIan@gmail.com>

* Fixed handling for a dbName that do not exist in the backup being restored (#148)

* Fixed handling for a dbName that do not exist in the backup being restored

* Fixed handling for a dbName that do not exist in the backup being restored

* validate: fix debug meta test ci (#153)

* validate: fix debug meta test ci

* *: extracts runBackup/runRestore in cmd into pkg/task (#156)

* *: extracts runBackup/runRestore in cmd into pkg/task

Defines a "Config" structure to store the parsed flags.

Use the "black-white-list" structure to define what tables/databases to
backup/restore.

* go.mod: update tidb to v4.0.0-beta

* restore: fix restore summary log (#150)

Co-authored-by: kennytm <kennytm@gmail.com>

* restore: enhance error handling (#152)

* restore: enhance error handling

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* unit test

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* fix region epoch error

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address comments

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* remove `Restore*`

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* address lint

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* add debug log

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* Apply suggestions from code review

Co-Authored-By: kennytm <kennytm@gmail.com>

* Update pkg/restore/import.go

Co-Authored-By: kennytm <kennytm@gmail.com>

* fix retry error

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

* handle RegionNotFound error

Signed-off-by: 5kbpers <tangminghua@pingcap.com>

Co-authored-by: Neil Shen <overvenus@gmail.com>
Co-authored-by: kennytm <kennytm@gmail.com>

Co-authored-by: 5kbpers <20279863+5kbpers@users.noreply.github.com>
Co-authored-by: Neil Shen <overvenus@gmail.com>
Co-authored-by: kennytm <kennytm@gmail.com>
Co-authored-by: disksing <i@disksing.com>
Co-authored-by: Ian <ArGregoryIan@gmail.com>
Co-authored-by: Kolbe Kegel <kolbe@kolbekegel.com>

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,29 @@
+---
+name: "🐛 Bug Report"
+about: Something isn't working as expected
+title: ''
+labels: 'bug'
+---
+
+Please answer these questions before submitting your issue. Thanks!
+
+1. What did you do?
+If possible, provide a recipe for reproducing the error.
+
+
+2. What did you expect to see?
+
+
+
+3. What did you see instead?
+
+
+
+4. What version of BR and TiDB/TiKV/PD are you using?
+
+<!--
+br -V
+tidb-server -V
+tikv-server -V
+pd-server -V
+-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,19 @@
+---
+name: "🚀 Feature Request"
+about: I have a suggestion
+labels: enhancement
+---
+
+## Feature Request
+
+### Describe your feature request related problem:
+<!-- A description of what the problem is. -->
+
+### Describe the feature you'd like:
+<!-- A description of what you want to happen. -->
+
+### Describe alternatives you've considered:
+<!-- A description of any alternative solutions or features you've considered. -->
+
+### Teachability, Documentation, Adoption, Migration Strategy:
+<!-- If you can, explain some scenarios how users might use this, or situations in which it would be helpful. Any API designs, mockups, or diagrams are also helpful. -->

.github/pull_request_template.md

@@ -0,0 +1,37 @@
+<!--
+Thank you for working on BR! Please read BR's [CONTRIBUTING](https://github.com/pingcap/br/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
+-->
+
+### What problem does this PR solve? <!--add issue link with summary if exists-->
+
+
+### What is changed and how it works?
+
+
+### Check List <!--REMOVE the items that are not applicable-->
+
+Tests <!-- At least one of them must be included. -->
+
+ - Unit test
+ - Integration test
+ - Manual test (add detailed scripts or steps below)
+ - No code
+
+Code changes
+
+ - Has exported function/method change
+ - Has exported variable/fields change
+ - Has interface methods change
+ - Has persistent data change
+
+Side effects
+
+ - Possible performance regression
+ - Increased code complexity
+ - Breaking backward compatibility
+
+Related changes
+
+ - Need to cherry-pick to the release branch
+ - Need to update the documentation
+ - Need to be included in the release note

.golangci.yml

@@ -9,7 +9,8 @@ issues:
       text: "Potential HTTP request made with variable url"
       linters:
         - gosec
-    - path: .go
-      text: "Use of weak random number generator"
+    # TODO Remove it.
+    - path: split_client.go
+      text: "SA1019:"
       linters:
-        - gosec
+        - staticcheck

CHANGELOG.md

@@ -0,0 +1,18 @@
+# BR (Backup and Restore) Change Log
+All notable changes to this project are documented in this file.
+See also,
+- [TiDB Changelog](https://github.com/pingcap/tidb/blob/master/CHANGELOG.md),
+- [TiKV Changelog](https://github.com/tikv/tikv/blob/master/CHANGELOG.md),
+- [PD Changelog](https://github.com/pingcap/pd/blob/master/CHANGELOG.md).
+
+## [3.1.0-beta.1] - 2020-01-10
+
+- Fix the inaccurate backup progress information [#127](https://github.com/pingcap/br/pull/127)
+- Improve the performance of splitting Regions [#122](https://github.com/pingcap/br/pull/122)
+- Add the backup and restore feature for partitioned tables [#137](https://github.com/pingcap/br/pull/137)
+- Add the feature of automatically scheduling PD schedulers [#123](https://github.com/pingcap/br/pull/123)
+- Fix the issue that data is overwritten after non `PKIsHandle` tables are restored [#139](https://github.com/pingcap/br/pull/139)
+
+## [3.1.0-beta] - 2019-12-20
+
+Initial release of the distributed backup and restore tool

Makefile

@@ -26,6 +26,8 @@ build_for_integration_test:
 		-o bin/br.test
 	# build key locker
 	GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go
+	# build gc
+	GO111MODULE=on go build -race -o bin/gc tests/br_z_gc_safepoint/*.go
 
 test:
 	GO111MODULE=on go test -race -tags leak ./...

cmd/backup.go

@@ -1,165 +1,21 @@
 package cmd
 
 import (
-	"context"
-
-	"github.com/pingcap/errors"
-	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/session"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 
-	"github.com/pingcap/br/pkg/backup"
-	"github.com/pingcap/br/pkg/storage"
 	"github.com/pingcap/br/pkg/summary"
+	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
 )
 
-const (
-	flagBackupTimeago     = "timeago"
-	flagBackupRateLimit   = "ratelimit"
-	flagBackupConcurrency = "concurrency"
-	flagBackupChecksum    = "checksum"
-	flagLastBackupTS      = "lastbackupts"
-)
-
-func defineBackupFlags(flagSet *pflag.FlagSet) {
-	flagSet.StringP(
-		flagBackupTimeago, "", "",
-		"The history version of the backup task, e.g. 1m, 1h. Do not exceed GCSafePoint")
-	flagSet.Uint64P(
-		flagBackupRateLimit, "", 0, "The rate limit of the backup task, MB/s per node")
-	flagSet.Uint32P(
-		flagBackupConcurrency, "", 4, "The size of thread pool on each node that execute the backup task")
-	flagSet.BoolP(flagBackupChecksum, "", true,
-		"Run checksum after backup")
-	flagSet.Uint64P(flagLastBackupTS, "", 0, "the last time backup ts")
-	_ = flagSet.MarkHidden(flagLastBackupTS)
-}
-
-func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error {
-	ctx, cancel := context.WithCancel(defaultContext)
-	defer cancel()
-
-	mgr, err := GetDefaultMgr()
-	if err != nil {
-		return err
-	}
-	defer mgr.Close()
-
-	timeago, err := flagSet.GetString(flagBackupTimeago)
-	if err != nil {
-		return err
-	}
-
-	ratelimit, err := flagSet.GetUint64(flagBackupRateLimit)
-	if err != nil {
-		return err
-	}
-
-	concurrency, err := flagSet.GetUint32(flagBackupConcurrency)
-	if err != nil {
-		return err
-	}
-	if concurrency == 0 {
-		err = errors.New("at least one thread required")
-		return err
-	}
-
-	checksum, err := flagSet.GetBool(flagBackupChecksum)
-	if err != nil {
-		return err
-	}
-
-	lastBackupTS, err := flagSet.GetUint64(flagLastBackupTS)
-	if err != nil {
-		return nil
-	}
-
-	u, err := storage.ParseBackendFromFlags(flagSet, FlagStorage)
-	if err != nil {
-		return err
-	}
-
-	client, err := backup.NewBackupClient(ctx, mgr)
-	if err != nil {
-		return nil
-	}
-
-	err = client.SetStorage(ctx, u)
-	if err != nil {
-		return err
-	}
-
-	backupTS, err := client.GetTS(ctx, timeago)
-	if err != nil {
-		return err
-	}
-
-	defer summary.Summary(cmdName)
-
-	ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema(
-		mgr.GetDomain(), mgr.GetTiKV(), backupTS, db, table)
-	if err != nil {
-		return err
-	}
-
-	// The number of regions need to backup
-	approximateRegions := 0
-	for _, r := range ranges {
-		var regionCount int
-		regionCount, err = mgr.GetRegionCount(ctx, r.StartKey, r.EndKey)
-		if err != nil {
-			return err
-		}
-		approximateRegions += regionCount
-	}
-
-	summary.CollectInt("backup total regions", approximateRegions)
-	// Backup
-	// Redirect to log if there is no log file to avoid unreadable output.
-	updateCh := utils.StartProgress(
-		ctx, cmdName, int64(approximateRegions), !HasLogFile())
-	err = client.BackupRanges(
-		ctx, ranges, lastBackupTS, backupTS, ratelimit, concurrency, updateCh)
-	if err != nil {
-		return err
-	}
-	// Backup has finished
-	close(updateCh)
-
-	// Checksum
-	backupSchemasConcurrency := backup.DefaultSchemaConcurrency
-	if backupSchemas.Len() < backupSchemasConcurrency {
-		backupSchemasConcurrency = backupSchemas.Len()
-	}
-	updateCh = utils.StartProgress(
-		ctx, "Checksum", int64(backupSchemas.Len()), !HasLogFile())
-	backupSchemas.SetSkipChecksum(!checksum)
-	backupSchemas.Start(
-		ctx, mgr.GetTiKV(), backupTS, uint(backupSchemasConcurrency), updateCh)
-
-	err = client.CompleteMeta(backupSchemas)
-	if err != nil {
-		return err
-	}
-
-	valid, err := client.FastChecksum()
-	if err != nil {
-		return err
-	}
-	if !valid {
-		log.Error("backup FastChecksum failed!")
-	}
-	// Checksum has finished
-	close(updateCh)
-
-	err = client.SaveBackupMeta(ctx)
-	if err != nil {
+func runBackupCommand(command *cobra.Command, cmdName string) error {
+	cfg := task.BackupConfig{Config: task.Config{LogProgress: HasLogFile()}}
+	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
 		return err
 	}
-	return nil
+	return task.RunBackup(GetDefaultContext(), cmdName, &cfg)
 }
 
 // NewBackupCommand return a full backup subcommand.
@@ -189,7 +45,7 @@ func NewBackupCommand() *cobra.Command {
 		newTableBackupCommand(),
 	)
 
-	defineBackupFlags(command.PersistentFlags())
+	task.DefineBackupFlags(command.PersistentFlags())
 	return command
 }
 
@@ -200,7 +56,7 @@ func newFullBackupCommand() *cobra.Command {
 		Short: "backup all database",
 		RunE: func(command *cobra.Command, _ []string) error {
 			// empty db/table means full backup.
-			return runBackup(command.Flags(), "Full backup", "", "")
+			return runBackupCommand(command, "Full backup")
 		},
 	}
 	return command
@@ -212,19 +68,10 @@ func newDbBackupCommand() *cobra.Command {
 		Use:   "db",
 		Short: "backup a database",
 		RunE: func(command *cobra.Command, _ []string) error {
-			db, err := command.Flags().GetString(flagDatabase)
-			if err != nil {
-				return err
-			}
-			if len(db) == 0 {
-				return errors.Errorf("empty database name is not allowed")
-			}
-			return runBackup(command.Flags(), "Database backup", db, "")
+			return runBackupCommand(command, "Database backup")
 		},
 	}
-	command.Flags().StringP(flagDatabase, "", "", "backup a table in the specific db")
-	_ = command.MarkFlagRequired(flagDatabase)
-
+	task.DefineDatabaseFlags(command)
 	return command
 }
 
@@ -234,26 +81,9 @@ func newTableBackupCommand() *cobra.Command {
 		Use:   "table",
 		Short: "backup a table",
 		RunE: func(command *cobra.Command, _ []string) error {
-			db, err := command.Flags().GetString(flagDatabase)
-			if err != nil {
-				return err
-			}
-			if len(db) == 0 {
-				return errors.Errorf("empty database name is not allowed")
-			}
-			table, err := command.Flags().GetString(flagTable)
-			if err != nil {
-				return err
-			}
-			if len(table) == 0 {
-				return errors.Errorf("empty table name is not allowed")
-			}
-			return runBackup(command.Flags(), "Table backup", db, table)
+			return runBackupCommand(command, "Table backup")
 		},
 	}
-	command.Flags().StringP(flagDatabase, "", "", "backup a table in the specific db")
-	command.Flags().StringP(flagTable, "t", "", "backup the specific table")
-	_ = command.MarkFlagRequired(flagDatabase)
-	_ = command.MarkFlagRequired(flagTable)
+	task.DefineTableFlags(command)
 	return command
 }