forked from vitessio/vitess
-
Notifications
You must be signed in to change notification settings - Fork 1
/
backup.go
580 lines (504 loc) · 15.7 KB
/
backup.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
// Copyright 2015, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mysqlctl
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"strings"
"sync"
log "github.com/golang/glog"
"golang.org/x/net/context"
"github.com/youtube/vitess/go/cgzip"
"github.com/youtube/vitess/go/sync2"
"github.com/youtube/vitess/go/vt/concurrency"
"github.com/youtube/vitess/go/vt/logutil"
"github.com/youtube/vitess/go/vt/mysqlctl/backupstorage"
"github.com/youtube/vitess/go/vt/mysqlctl/proto"
)
// This file handles the backup and restore related code
const (
// the three bases for files to restore
backupInnodbDataHomeDir = "InnoDBData"
backupInnodbLogGroupHomeDir = "InnoDBLog"
backupData = "Data"
// the manifest file name
backupManifest = "MANIFEST"
)
const (
// slaveStartDeadline is the deadline for starting a slave
slaveStartDeadline = 30
)
var (
// ErrNoBackup is returned when there is no backup
ErrNoBackup = errors.New("no available backup")
)
// FileEntry is one file to backup
type FileEntry struct {
// Base is one of:
// - backupInnodbDataHomeDir for files that go into Mycnf.InnodbDataHomeDir
// - backupInnodbLogGroupHomeDir for files that go into Mycnf.InnodbLogGroupHomeDir
// - backupData for files that go into Mycnf.DataDir
Base string
// Name is the file name, relative to Base
Name string
// Hash is the hash of the gzip compressed data stored in the
// BackupStorage.
Hash string
}
func (fe *FileEntry) open(cnf *Mycnf, readOnly bool) (*os.File, error) {
// find the root to use
var root string
switch fe.Base {
case backupInnodbDataHomeDir:
root = cnf.InnodbDataHomeDir
case backupInnodbLogGroupHomeDir:
root = cnf.InnodbLogGroupHomeDir
case backupData:
root = cnf.DataDir
default:
return nil, fmt.Errorf("unknown base: %v", fe.Base)
}
// and open the file
name := path.Join(root, fe.Name)
var fd *os.File
var err error
if readOnly {
fd, err = os.Open(name)
} else {
dir := path.Dir(name)
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return nil, fmt.Errorf("cannot create destination directory %v: %v", dir, err)
}
fd, err = os.Create(name)
}
if err != nil {
return nil, fmt.Errorf("cannot open source file %v: %v", name, err)
}
return fd, nil
}
// BackupManifest represents the backup. It lists all the files, and
// the ReplicationPosition that the backup was taken at.
type BackupManifest struct {
// FileEntries contains all the files in the backup
FileEntries []FileEntry
// ReplicationPosition is the position at which the backup was taken
ReplicationPosition proto.ReplicationPosition
}
// isDbDir returns true if the given directory contains a DB
func isDbDir(p string) bool {
// db.opt is there
if _, err := os.Stat(path.Join(p, "db.opt")); err == nil {
return true
}
// Look for at least one .frm file
fis, err := ioutil.ReadDir(p)
if err != nil {
return false
}
for _, fi := range fis {
if strings.HasSuffix(fi.Name(), ".frm") {
return true
}
}
return false
}
func addDirectory(fes []FileEntry, base string, baseDir string, subDir string) ([]FileEntry, error) {
p := path.Join(baseDir, subDir)
fis, err := ioutil.ReadDir(p)
if err != nil {
return nil, err
}
for _, fi := range fis {
fes = append(fes, FileEntry{
Base: base,
Name: path.Join(subDir, fi.Name()),
})
}
return fes, nil
}
func findFilesTobackup(cnf *Mycnf) ([]FileEntry, error) {
var err error
var result []FileEntry
// first add inno db files
result, err = addDirectory(result, backupInnodbDataHomeDir, cnf.InnodbDataHomeDir, "")
if err != nil {
return nil, err
}
result, err = addDirectory(result, backupInnodbLogGroupHomeDir, cnf.InnodbLogGroupHomeDir, "")
if err != nil {
return nil, err
}
// then add DB directories
fis, err := ioutil.ReadDir(cnf.DataDir)
if err != nil {
return nil, err
}
for _, fi := range fis {
p := path.Join(cnf.DataDir, fi.Name())
if isDbDir(p) {
result, err = addDirectory(result, backupData, cnf.DataDir, fi.Name())
if err != nil {
return nil, err
}
}
}
return result, nil
}
// Backup is the main entry point for a backup:
// - uses the BackupStorage service to store a new backup
// - shuts down Mysqld during the backup
// - remember if we were replicating, restore the exact same state
func Backup(ctx context.Context, mysqld MysqlDaemon, logger logutil.Logger, bucket, name string, backupConcurrency int, hookExtraEnv map[string]string) error {
// start the backup with the BackupStorage
bs, err := backupstorage.GetBackupStorage()
if err != nil {
return err
}
bh, err := bs.StartBackup(bucket, name)
if err != nil {
return fmt.Errorf("StartBackup failed: %v", err)
}
if err = backup(ctx, mysqld, logger, bh, backupConcurrency, hookExtraEnv); err != nil {
if abortErr := bh.AbortBackup(); abortErr != nil {
logger.Errorf("failed to abort backup: %v", abortErr)
}
return err
}
return bh.EndBackup()
}
func backup(ctx context.Context, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, backupConcurrency int, hookExtraEnv map[string]string) error {
// save initial state so we can restore
slaveStartRequired := false
sourceIsMaster := false
readOnly := true
var replicationPosition proto.ReplicationPosition
// see if we need to restart replication after backup
logger.Infof("getting current replication status")
slaveStatus, err := mysqld.SlaveStatus()
switch err {
case nil:
slaveStartRequired = slaveStatus.SlaveRunning()
case ErrNotSlave:
// keep going if we're the master, might be a degenerate case
sourceIsMaster = true
default:
return fmt.Errorf("cannot get slave status: %v", err)
}
// get the read-only flag
readOnly, err = mysqld.IsReadOnly()
if err != nil {
return fmt.Errorf("cannot get read only status: %v", err)
}
// get the replication position
if sourceIsMaster {
if !readOnly {
logger.Infof("turning master read-onyl before backup")
if err = mysqld.SetReadOnly(true); err != nil {
return fmt.Errorf("cannot get read only status: %v", err)
}
}
replicationPosition, err = mysqld.MasterPosition()
if err != nil {
return fmt.Errorf("cannot get master position: %v", err)
}
} else {
if err = StopSlave(mysqld, hookExtraEnv); err != nil {
return fmt.Errorf("cannot stop slave: %v", err)
}
var slaveStatus proto.ReplicationStatus
slaveStatus, err = mysqld.SlaveStatus()
if err != nil {
return fmt.Errorf("cannot get slave status: %v", err)
}
replicationPosition = slaveStatus.Position
}
logger.Infof("using replication position: %v", replicationPosition)
// shutdown mysqld
err = mysqld.Shutdown(ctx, true)
if err != nil {
return fmt.Errorf("cannot shutdown mysqld: %v", err)
}
// get the files to backup
fes, err := findFilesTobackup(mysqld.Cnf())
if err != nil {
return fmt.Errorf("cannot find files to backup: %v", err)
}
logger.Infof("found %v files to backup", len(fes))
// backup everything
if err := backupFiles(mysqld, logger, bh, fes, replicationPosition, backupConcurrency); err != nil {
return fmt.Errorf("cannot backup files: %v", err)
}
// Try to restart mysqld
err = mysqld.Start(ctx)
if err != nil {
return fmt.Errorf("cannot restart mysqld: %v", err)
}
// Restore original mysqld state that we saved above.
if slaveStartRequired {
logger.Infof("restarting mysql replication")
if err := StartSlave(mysqld, hookExtraEnv); err != nil {
return fmt.Errorf("cannot restart slave: %v", err)
}
// this should be quick, but we might as well just wait
if err := WaitForSlaveStart(mysqld, slaveStartDeadline); err != nil {
return fmt.Errorf("slave is not restarting: %v", err)
}
}
// And set read-only mode
logger.Infof("resetting mysqld read-only to %v", readOnly)
if err := mysqld.SetReadOnly(readOnly); err != nil {
return err
}
return nil
}
func backupFiles(mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fes []FileEntry, replicationPosition proto.ReplicationPosition, backupConcurrency int) (err error) {
sema := sync2.NewSemaphore(backupConcurrency, 0)
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for i, fe := range fes {
wg.Add(1)
go func(i int, fe FileEntry) {
defer wg.Done()
// wait until we are ready to go, skip if we already
// encountered an error
sema.Acquire()
defer sema.Release()
if rec.HasErrors() {
return
}
// open the source file for reading
source, err := fe.open(mysqld.Cnf(), true)
if err != nil {
rec.RecordError(err)
return
}
defer source.Close()
// open the destination file for writing, and a buffer
name := fmt.Sprintf("%v", i)
wc, err := bh.AddFile(name)
if err != nil {
rec.RecordError(fmt.Errorf("cannot add file: %v", err))
return
}
defer func() { rec.RecordError(wc.Close()) }()
dst := bufio.NewWriterSize(wc, 2*1024*1024)
// create the hasher and the tee on top
hasher := newHasher()
tee := io.MultiWriter(dst, hasher)
// create the gzip compression filter
gzip, err := cgzip.NewWriterLevel(tee, cgzip.Z_BEST_SPEED)
if err != nil {
rec.RecordError(fmt.Errorf("cannot create gziper: %v", err))
return
}
// copy from the source file to gzip to tee to output file and hasher
_, err = io.Copy(gzip, source)
if err != nil {
rec.RecordError(fmt.Errorf("cannot copy data: %v", err))
return
}
// close gzip to flush it, after that the hash is good
if err = gzip.Close(); err != nil {
rec.RecordError(fmt.Errorf("cannot close gzip: %v", err))
return
}
// flush the buffer to finish writing, save the hash
rec.RecordError(dst.Flush())
fes[i].Hash = hasher.HashString()
}(i, fe)
}
wg.Wait()
if rec.HasErrors() {
return rec.Error()
}
// open the MANIFEST
wc, err := bh.AddFile(backupManifest)
if err != nil {
return fmt.Errorf("cannot add %v to backup: %v", backupManifest, err)
}
defer func() {
if closeErr := wc.Close(); err == nil {
err = closeErr
}
}()
// JSON-encode and write the MANIFEST
bm := &BackupManifest{
FileEntries: fes,
ReplicationPosition: replicationPosition,
}
data, err := json.MarshalIndent(bm, "", " ")
if err != nil {
return fmt.Errorf("cannot JSON encode %v: %v", backupManifest, err)
}
if _, err := wc.Write([]byte(data)); err != nil {
return fmt.Errorf("cannot write %v: %v", backupManifest, err)
}
return nil
}
// checkNoDB makes sure there is no vt_ db already there. Used by Restore,
// we do not wnat to destroy an existing DB.
func checkNoDB(mysqld MysqlDaemon) error {
qr, err := mysqld.FetchSuperQuery("SHOW DATABASES")
if err != nil {
return fmt.Errorf("checkNoDB failed: %v", err)
}
for _, row := range qr.Rows {
if strings.HasPrefix(row[0].String(), "vt_") {
dbName := row[0].String()
tableQr, err := mysqld.FetchSuperQuery("SHOW TABLES FROM " + dbName)
if err != nil {
return fmt.Errorf("checkNoDB failed: %v", err)
} else if len(tableQr.Rows) == 0 {
// no tables == empty db, all is well
continue
}
return fmt.Errorf("checkNoDB failed, found active db %v", dbName)
}
}
return nil
}
// restoreFiles will copy all the files from the BackupStorage to the
// right place
func restoreFiles(cnf *Mycnf, bh backupstorage.BackupHandle, fes []FileEntry, restoreConcurrency int) error {
sema := sync2.NewSemaphore(restoreConcurrency, 0)
rec := concurrency.AllErrorRecorder{}
wg := sync.WaitGroup{}
for i, fe := range fes {
wg.Add(1)
go func(i int, fe FileEntry) {
defer wg.Done()
// wait until we are ready to go, skip if we already
// encountered an error
sema.Acquire()
defer sema.Release()
if rec.HasErrors() {
return
}
// open the source file for reading
name := fmt.Sprintf("%v", i)
source, err := bh.ReadFile(name)
if err != nil {
rec.RecordError(err)
return
}
defer source.Close()
// open the destination file for writing
dstFile, err := fe.open(cnf, false)
if err != nil {
rec.RecordError(err)
return
}
defer func() { rec.RecordError(dstFile.Close()) }()
// create a buffering output
dst := bufio.NewWriterSize(dstFile, 2*1024*1024)
// create hash to write the compressed data to
hasher := newHasher()
// create a Tee: we split the input into the hasher
// and into the gunziper
tee := io.TeeReader(source, hasher)
// create the uncompresser
gz, err := cgzip.NewReader(tee)
if err != nil {
rec.RecordError(err)
return
}
defer func() { rec.RecordError(gz.Close()) }()
// copy the data. Will also write to the hasher
if _, err = io.Copy(dst, gz); err != nil {
rec.RecordError(err)
return
}
// check the hash
hash := hasher.HashString()
if hash != fe.Hash {
rec.RecordError(fmt.Errorf("hash mismatch for %v, got %v expected %v", fe.Name, hash, fe.Hash))
return
}
// flush the buffer
rec.RecordError(dst.Flush())
}(i, fe)
}
wg.Wait()
return rec.Error()
}
// Restore is the main entry point for backup restore. If there is no
// appropriate backup on the BackupStorage, Restore logs an error
// and returns ErrNoBackup. Any other error is returned.
func Restore(ctx context.Context, mysqld MysqlDaemon, bucket string, restoreConcurrency int, hookExtraEnv map[string]string) (proto.ReplicationPosition, error) {
// find the right backup handle: most recent one, with a MANIFEST
log.Infof("Restore: looking for a suitable backup to restore")
bs, err := backupstorage.GetBackupStorage()
if err != nil {
return proto.ReplicationPosition{}, err
}
bhs, err := bs.ListBackups(bucket)
if err != nil {
return proto.ReplicationPosition{}, fmt.Errorf("ListBackups failed: %v", err)
}
toRestore := len(bhs) - 1
var bh backupstorage.BackupHandle
var bm BackupManifest
for toRestore >= 0 {
bh = bhs[toRestore]
if rc, err := bh.ReadFile(backupManifest); err == nil {
dec := json.NewDecoder(rc)
err := dec.Decode(&bm)
rc.Close()
if err != nil {
log.Warningf("Possibly incomplete backup %v in bucket %v on BackupStorage (cannot JSON decode MANIFEST: %v)", bh.Name(), bucket, err)
} else {
log.Infof("Restore: found backup %v %v to restore with %v files", bh.Bucket(), bh.Name(), len(bm.FileEntries))
break
}
} else {
log.Warningf("Possibly incomplete backup %v in bucket %v on BackupStorage (cannot read MANIFEST)", bh.Name(), bucket)
}
toRestore--
}
if toRestore < 0 {
log.Errorf("No backup to restore on BackupStorage for bucket %v", bucket)
return proto.ReplicationPosition{}, ErrNoBackup
}
log.Infof("Restore: checking no existing data is present")
if err := checkNoDB(mysqld); err != nil {
return proto.ReplicationPosition{}, err
}
log.Infof("Restore: shutdown mysqld")
err = mysqld.Shutdown(ctx, true)
if err != nil {
return proto.ReplicationPosition{}, err
}
log.Infof("Restore: copying all files")
if err := restoreFiles(mysqld.Cnf(), bh, bm.FileEntries, restoreConcurrency); err != nil {
return proto.ReplicationPosition{}, err
}
// mysqld needs to be running in order for mysql_upgrade to work.
log.Infof("Restore: starting mysqld for mysql_upgrade")
err = mysqld.Start(ctx)
if err != nil {
return proto.ReplicationPosition{}, err
}
log.Infof("Restore: running mysql_upgrade")
if err := mysqld.RunMysqlUpgrade(); err != nil {
return proto.ReplicationPosition{}, fmt.Errorf("mysql_upgrade failed: %v", err)
}
// The MySQL manual recommends restarting mysqld after running mysql_upgrade,
// so that any changes made to system tables take effect.
log.Infof("Restore: restarting mysqld after mysql_upgrade")
err = mysqld.Shutdown(ctx, true)
if err != nil {
return proto.ReplicationPosition{}, err
}
err = mysqld.Start(ctx)
if err != nil {
return proto.ReplicationPosition{}, err
}
return bm.ReplicationPosition, nil
}