go.mod: update dependencies

Makefile

@@ -68,20 +68,30 @@ static: tools
 	tools/bin/goimports -w -d -format-only -local $(BR_PKG) $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
 	tools/bin/govet --shadow $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
 
+	@# why some lints are disabled?
+	@#   gochecknoglobals - disabled because we do use quite a lot of globals
+	@#          goimports - executed above already
+	@#              gofmt - ditto
+	@#                wsl - too pedantic about the formatting
+	@#             funlen - PENDING REFACTORING
+	@#           gocognit - PENDING REFACTORING
+	@#              godox - TODO
+	@#              gomnd - too many magic numbers, and too pedantic (even 2*x got flagged...)
+	@#        testpackage - several test packages still rely on private functions
+	@#             nestif - PENDING REFACTORING
+	@#           goerr113 - it mistaken pingcap/errors with standard errors
 	CGO_ENABLED=0 tools/bin/golangci-lint run --enable-all --deadline 120s \
 		--disable gochecknoglobals \
-		--disable gochecknoinits \
-		--disable interfacer \
 		--disable goimports \
 		--disable gofmt \
 		--disable wsl \
 		--disable funlen \
-		--disable whitespace \
 		--disable gocognit \
 		--disable godox \
 		--disable gomnd \
 		--disable testpackage \
 		--disable nestif \
+		--disable goerr113 \
 		$$($(PACKAGE_DIRECTORIES))
 
 lint: tools

go.mod

@@ -3,40 +3,31 @@ module github.com/pingcap/br
 go 1.13
 
 require (
-	cloud.google.com/go/storage v1.4.0
-	github.com/aws/aws-sdk-go v1.26.1
-	github.com/cheggaaa/pb/v3 v3.0.1
-	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
-	github.com/fsouza/fake-gcs-server v1.15.0
-	github.com/go-sql-driver/mysql v1.4.1
+	cloud.google.com/go/storage v1.5.0
+	github.com/aws/aws-sdk-go v1.30.24
+	github.com/cheggaaa/pb/v3 v3.0.4
+	github.com/fsouza/fake-gcs-server v1.17.0
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gogo/protobuf v1.3.1
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
-	github.com/klauspost/cpuid v1.2.0 // indirect
-	github.com/montanaflynn/stats v0.5.0 // indirect
-	github.com/onsi/ginkgo v1.11.0 // indirect
-	github.com/onsi/gomega v1.8.1 // indirect
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
 	github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
-	github.com/pingcap/kvproto v0.0.0-20200423020121-038e31959c2a
+	github.com/pingcap/kvproto v0.0.0-20200509065137-6a4d5c264a8b
 	github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
-	github.com/pingcap/parser v0.0.0-20200424075042-8222d8b724a4
-	github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2
-	github.com/pingcap/tidb v1.1.0-beta.0.20200424154252-5ede18f10eed
+	github.com/pingcap/parser v0.0.0-20200507022230-f3bf29096657
+	github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200508085806-feb738d721fb
+	github.com/pingcap/tidb v1.1.0-beta.0.20200509133407-a9dc72cf2558
 	github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible
 	github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee
-	github.com/prometheus/client_golang v1.0.0
-	github.com/prometheus/common v0.4.1
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.5
+	github.com/prometheus/client_golang v1.5.1
+	github.com/prometheus/common v0.9.1
+	github.com/sirupsen/logrus v1.6.0
+	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
-	github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
-	go.opencensus.io v0.22.2 // indirect
-	go.uber.org/zap v1.14.1
-	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	google.golang.org/api v0.14.0
-	google.golang.org/grpc v1.25.1
+	go.uber.org/zap v1.15.0
+	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6
+	google.golang.org/api v0.15.1
+	google.golang.org/grpc v1.26.0
 )

pkg/backup/client.go

@@ -169,7 +169,9 @@ func (bc *Client) SaveBackupMeta(ctx context.Context, ddlJobs []*model.Job) erro
 	return bc.storage.Write(ctx, utils.MetaFile, backupMetaData)
 }
 
-func buildTableRanges(tbl *model.TableInfo) ([]kv.KeyRange, error) {
+// BuildTableRanges returns the key ranges encompassing the entire table,
+// and its partitions if exists.
+func BuildTableRanges(tbl *model.TableInfo) ([]kv.KeyRange, error) {
 	pis := tbl.GetPartitionInfo()
 	if pis == nil {
 		// Short path, no partition.
@@ -202,7 +204,6 @@ func appendRanges(tbl *model.TableInfo, tblID int64) ([]kv.KeyRange, error) {
 		kvRanges = append(kvRanges, idxRanges...)
 	}
 	return kvRanges, nil
-
 }
 
 // BuildBackupRangeAndSchema gets the range and schema of tables.
@@ -285,7 +286,7 @@ func BuildBackupRangeAndSchema(
 			}
 			backupSchemas.pushPending(schema, dbInfo.Name.L, tableInfo.Name.L)
 
-			tableRanges, err := buildTableRanges(tableInfo)
+			tableRanges, err := BuildTableRanges(tableInfo)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -897,7 +898,7 @@ func (bc *Client) CollectChecksums() ([]Checksum, error) {
 
 // CompleteMeta wait response of admin checksum from TiDB to complete backup meta.
 func (bc *Client) CompleteMeta(backupSchemas *Schemas) error {
-	schemas, err := backupSchemas.finishTableChecksum()
+	schemas, err := backupSchemas.FinishTableChecksum()
 	if err != nil {
 		return err
 	}

pkg/backup/client_test.go

@@ -1,6 +1,6 @@
 // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
 
-package backup
+package backup_test
 
 import (
 	"context"
@@ -10,20 +10,23 @@ import (
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/parser/model"
+	pd "github.com/pingcap/pd/v4/client"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/store/mockstore/mocktikv"
 	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/util/codec"
 
+	"github.com/pingcap/br/pkg/backup"
 	"github.com/pingcap/br/pkg/conn"
 )
 
 type testBackup struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 
-	backupClient *Client
+	mockPDClient pd.Client
+	backupClient *backup.Client
 }
 
 var _ = Suite(&testBackup{})
@@ -33,15 +36,14 @@ func TestT(t *testing.T) {
 }
 
 func (r *testBackup) SetUpSuite(c *C) {
-	mockPDClient := mocktikv.NewPDClient(mocktikv.NewCluster())
+	r.mockPDClient = mocktikv.NewPDClient(mocktikv.NewCluster())
 	r.ctx, r.cancel = context.WithCancel(context.Background())
 	mockMgr := &conn.Mgr{}
-	mockMgr.SetPDClient(mockPDClient)
+	mockMgr.SetPDClient(r.mockPDClient)
 	mockMgr.SetPDHTTP([]string{"test"}, nil)
-	r.backupClient = &Client{
-		clusterID: mockPDClient.GetClusterID(r.ctx),
-		mgr:       mockMgr,
-	}
+	var err error
+	r.backupClient, err = backup.NewBackupClient(r.ctx, mockMgr)
+	c.Assert(err, IsNil)
 }
 
 func (r *testBackup) TestGetTS(c *C) {
@@ -81,10 +83,10 @@ func (r *testBackup) TestGetTS(c *C) {
 	c.Assert(err, ErrorMatches, "backup ts overflow.*")
 
 	// timeago = "10h" exceed GCSafePoint
-	p, l, err := r.backupClient.mgr.GetPDClient().GetTS(r.ctx)
+	p, l, err := r.mockPDClient.GetTS(r.ctx)
 	c.Assert(err, IsNil)
 	now := oracle.ComposeTS(p, l)
-	_, err = r.backupClient.mgr.GetPDClient().UpdateGCSafePoint(r.ctx, now)
+	_, err = r.mockPDClient.UpdateGCSafePoint(r.ctx, now)
 	c.Assert(err, IsNil)
 	_, err = r.backupClient.GetTS(r.ctx, 10*time.Hour, 0)
 	c.Assert(err, ErrorMatches, "GC safepoint [0-9]+ exceed TS [0-9]+")
@@ -94,7 +96,6 @@ func (r *testBackup) TestGetTS(c *C) {
 	ts, err = r.backupClient.GetTS(r.ctx, time.Minute, backupts)
 	c.Assert(err, IsNil)
 	c.Assert(ts, Equals, backupts)
-
 }
 
 func (r *testBackup) TestBuildTableRange(c *C) {
@@ -125,16 +126,15 @@ func (r *testBackup) TestBuildTableRange(c *C) {
 			tbl.Partition.Definitions = append(tbl.Partition.Definitions,
 				model.PartitionDefinition{ID: id})
 		}
-		ranges, err := buildTableRanges(tbl)
+		ranges, err := backup.BuildTableRanges(tbl)
 		c.Assert(err, IsNil)
 		c.Assert(ranges, DeepEquals, cs.trs)
 	}
 
 	tbl := &model.TableInfo{ID: 7}
-	ranges, err := buildTableRanges(tbl)
+	ranges, err := backup.BuildTableRanges(tbl)
 	c.Assert(err, IsNil)
 	c.Assert(ranges, DeepEquals, []kv.KeyRange{
 		{StartKey: tablecodec.EncodeRowKey(7, low), EndKey: tablecodec.EncodeRowKey(7, high)},
 	})
-
 }

pkg/backup/metrics.go

@@ -25,7 +25,7 @@ var (
 		})
 )
 
-func init() {
+func init() { // nolint:gochecknoinits
 	prometheus.MustRegister(backupRegionCounters)
 	prometheus.MustRegister(backupRegionHistogram)
 }

pkg/backup/safe_point_test.go

@@ -1,6 +1,6 @@
 // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
 
-package backup
+package backup_test
 
 import (
 	"context"
@@ -10,6 +10,7 @@ import (
 	pd "github.com/pingcap/pd/v4/client"
 	"github.com/pingcap/tidb/util/testleak"
 
+	"github.com/pingcap/br/pkg/backup"
 	"github.com/pingcap/br/pkg/mock"
 )
 
@@ -36,19 +37,19 @@ func (s *testSafePointSuite) TestCheckGCSafepoint(c *C) {
 	ctx := context.Background()
 	pdClient := &mockSafePoint{Client: s.mock.PDClient, safepoint: 2333}
 	{
-		err := CheckGCSafePoint(ctx, pdClient, 2333+1)
+		err := backup.CheckGCSafePoint(ctx, pdClient, 2333+1)
 		c.Assert(err, IsNil)
 	}
 	{
-		err := CheckGCSafePoint(ctx, pdClient, 2333)
+		err := backup.CheckGCSafePoint(ctx, pdClient, 2333)
 		c.Assert(err, NotNil)
 	}
 	{
-		err := CheckGCSafePoint(ctx, pdClient, 2333-1)
+		err := backup.CheckGCSafePoint(ctx, pdClient, 2333-1)
 		c.Assert(err, NotNil)
 	}
 	{
-		err := CheckGCSafePoint(ctx, pdClient, 0)
+		err := backup.CheckGCSafePoint(ctx, pdClient, 0)
 		c.Assert(err, ErrorMatches, "GC safepoint 2333 exceed TS 0")
 	}
 }

pkg/backup/schema.go

@@ -29,7 +29,7 @@ const (
 	DefaultSchemaConcurrency = 64
 )
 
-// Schemas is task for backuping schemas
+// Schemas is task for backuping schemas.
 type Schemas struct {
 	// name -> schema
 	schemas        map[string]backup.Schema
@@ -110,7 +110,8 @@ func (pending *Schemas) Start(
 	}()
 }
 
-func (pending *Schemas) finishTableChecksum() ([]*backup.Schema, error) {
+// FinishTableChecksum waits until all schemas' checksums are verified.
+func (pending *Schemas) FinishTableChecksum() ([]*backup.Schema, error) {
 	schemas := make([]*backup.Schema, 0, len(pending.schemas))
 	for {
 		select {

pkg/backup/schema_test.go

@@ -1,6 +1,6 @@
 // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
 
-package backup
+package backup_test
 
 import (
 	"context"
@@ -12,6 +12,7 @@ import (
 	"github.com/pingcap/tidb/util/testkit"
 	"github.com/pingcap/tidb/util/testleak"
 
+	"github.com/pingcap/br/pkg/backup"
 	"github.com/pingcap/br/pkg/mock"
 )
 
@@ -60,7 +61,7 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 		DoTables: []*filter.Table{{Schema: "test", Name: "t1"}},
 	})
 	c.Assert(err, IsNil)
-	_, backupSchemas, err := BuildBackupRangeAndSchema(
+	_, backupSchemas, err := backup.BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, testFilter, math.MaxUint64)
 	c.Assert(err, IsNil)
 	c.Assert(backupSchemas, IsNil)
@@ -70,15 +71,15 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 		DoTables: []*filter.Table{{Schema: "foo", Name: "t1"}},
 	})
 	c.Assert(err, IsNil)
-	_, backupSchemas, err = BuildBackupRangeAndSchema(
+	_, backupSchemas, err = backup.BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, fooFilter, math.MaxUint64)
 	c.Assert(err, IsNil)
 	c.Assert(backupSchemas, IsNil)
 
 	// Empty database.
 	noFilter, err := filter.New(false, &filter.Rules{})
 	c.Assert(err, IsNil)
-	_, backupSchemas, err = BuildBackupRangeAndSchema(
+	_, backupSchemas, err = backup.BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, noFilter, math.MaxUint64)
 	c.Assert(err, IsNil)
 	c.Assert(backupSchemas, IsNil)
@@ -88,13 +89,13 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 	tk.MustExec("create table t1 (a int);")
 	tk.MustExec("insert into t1 values (10);")
 
-	_, backupSchemas, err = BuildBackupRangeAndSchema(
+	_, backupSchemas, err = backup.BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, testFilter, math.MaxUint64)
 	c.Assert(err, IsNil)
 	c.Assert(backupSchemas.Len(), Equals, 1)
 	updateCh := new(simpleProgress)
 	backupSchemas.Start(context.Background(), s.mock.Storage, math.MaxUint64, 1, updateCh)
-	schemas, err := backupSchemas.finishTableChecksum()
+	schemas, err := backupSchemas.FinishTableChecksum()
 	c.Assert(updateCh.get(), Equals, int64(1))
 	c.Assert(err, IsNil)
 	c.Assert(len(schemas), Equals, 1)
@@ -108,13 +109,13 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 	tk.MustExec("insert into t2 values (10);")
 	tk.MustExec("insert into t2 values (11);")
 
-	_, backupSchemas, err = BuildBackupRangeAndSchema(
+	_, backupSchemas, err = backup.BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, noFilter, math.MaxUint64)
 	c.Assert(err, IsNil)
 	c.Assert(backupSchemas.Len(), Equals, 2)
 	updateCh.reset()
 	backupSchemas.Start(context.Background(), s.mock.Storage, math.MaxUint64, 2, updateCh)
-	schemas, err = backupSchemas.finishTableChecksum()
+	schemas, err = backupSchemas.FinishTableChecksum()
 	c.Assert(updateCh.get(), Equals, int64(2))
 	c.Assert(err, IsNil)
 	c.Assert(len(schemas), Equals, 2)