forked from pingcap/br
/
push.go
120 lines (107 loc) · 3 KB
/
push.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
package backup
import (
"context"
"sync"
"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/backup"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
"go.uber.org/zap"
berrors "github.com/pingcap/br/pkg/errors"
"github.com/pingcap/br/pkg/glue"
"github.com/pingcap/br/pkg/rtree"
)
// pushDown wraps a backup task.
type pushDown struct {
mgr ClientMgr
respCh chan *backup.BackupResponse
errCh chan error
}
// newPushDown creates a push down backup.
func newPushDown(mgr ClientMgr, cap int) *pushDown {
return &pushDown{
mgr: mgr,
respCh: make(chan *backup.BackupResponse, cap),
errCh: make(chan error, cap),
}
}
// FullBackup make a full backup of a tikv cluster.
func (push *pushDown) pushBackup(
ctx context.Context,
req backup.BackupRequest,
stores []*metapb.Store,
updateCh glue.Progress,
) (rtree.RangeTree, error) {
// Push down backup tasks to all tikv instances.
res := rtree.NewRangeTree()
wg := new(sync.WaitGroup)
for _, s := range stores {
storeID := s.GetId()
if s.GetState() != metapb.StoreState_Up {
log.Warn("skip store", zap.Uint64("StoreID", storeID), zap.Stringer("State", s.GetState()))
continue
}
client, err := push.mgr.GetBackupClient(ctx, storeID)
if err != nil {
log.Error("fail to connect store", zap.Uint64("StoreID", storeID))
return res, errors.Trace(err)
}
wg.Add(1)
go func() {
defer wg.Done()
err := SendBackup(
ctx, storeID, client, req,
func(resp *backup.BackupResponse) error {
// Forward all responses (including error).
push.respCh <- resp
return nil
},
func() (backup.BackupClient, error) {
log.Warn("reset the connection in push", zap.Uint64("storeID", storeID))
return push.mgr.ResetBackupClient(ctx, storeID)
})
if err != nil {
push.errCh <- err
return
}
}()
}
go func() {
wg.Wait()
// TODO: test concurrent receive response and close channel.
close(push.respCh)
}()
for {
select {
case resp, ok := <-push.respCh:
if !ok {
// Finished.
return res, nil
}
if resp.GetError() == nil {
// None error means range has been backuped successfully.
res.Put(
resp.GetStartKey(), resp.GetEndKey(), resp.GetFiles())
// Update progress
updateCh.Inc()
} else {
errPb := resp.GetError()
switch v := errPb.Detail.(type) {
case *backup.Error_KvError:
log.Warn("backup occur kv error", zap.Reflect("error", v))
case *backup.Error_RegionError:
log.Warn("backup occur region error", zap.Reflect("error", v))
case *backup.Error_ClusterIdError:
log.Error("backup occur cluster ID error", zap.Reflect("error", v))
return res, errors.Annotatef(berrors.ErrKVClusterIDMismatch, "%v", errPb)
default:
log.Error("backup occur unknown error", zap.String("error", errPb.GetMsg()))
return res, errors.Annotatef(berrors.ErrKVUnknown, "%v", errPb)
}
}
case err := <-push.errCh:
return res, errors.Trace(err)
}
}
}