Skip to content

Commit a8011e0

Browse files
boris-ilijicboris-ilijic
authored
Merge pull request #1139 from percona/PBM-1511-fallback-config-params
PBM-1511: Configuration for Fallback dbpath feature
2 parents 8634aff + 82e9176 commit a8011e0

File tree

12 files changed

+603
-61
lines changed

12 files changed

+603
-61
lines changed

‎cmd/pbm-agent/restore.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,8 +180,25 @@ func (a *Agent) Restore(ctx context.Context, r *ctrl.RestoreCmd, opid ctrl.OPID,
180180
lck = nil
181181
}
182182

183+
fallbackOpt := cfg.Restore.GetFallbackEnabled()
184+
if r.Fallback != nil {
185+
fallbackOpt = *r.Fallback
186+
}
187+
allowPartlyDoneOpt := cfg.Restore.GetAllowPartlyDone()
188+
if r.AllowPartlyDone != nil {
189+
allowPartlyDoneOpt = *r.AllowPartlyDone
190+
}
191+
183192
var rstr *restore.PhysRestore
184-
rstr, err = restore.NewPhysical(ctx, a.leadConn, a.nodeConn, nodeInfo, r.RSMap)
193+
rstr, err = restore.NewPhysical(
194+
ctx,
195+
a.leadConn,
196+
a.nodeConn,
197+
nodeInfo,
198+
r.RSMap,
199+
fallbackOpt,
200+
allowPartlyDoneOpt,
201+
)
185202
if err != nil {
186203
l.Error("init physical backup: %v", err)
187204
return

‎cmd/pbm/backup.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,9 @@ type bcpDesc struct {
329329
PBMVersion string `json:"pbm_version" yaml:"pbm_version"`
330330
Status defs.Status `json:"status" yaml:"status"`
331331
Size int64 `json:"size" yaml:"-"`
332+
SizeUncompressed int64 `json:"size_uncompressed" yaml:"-"`
332333
HSize string `json:"size_h" yaml:"size_h"`
334+
HSizeUncompressed string `json:"size_uncompressed_h" yaml:"size_uncompressed_h"`
333335
StorageName string `json:"storage_name,omitempty" yaml:"storage_name,omitempty"`
334336
Err *string `json:"error,omitempty" yaml:"error,omitempty"`
335337
Replsets []bcpReplDesc `json:"replsets" yaml:"replsets"`
@@ -340,6 +342,10 @@ type bcpReplDesc struct {
340342
Status defs.Status `json:"status" yaml:"status"`
341343
Node string `json:"node" yaml:"node"`
342344
Files []backup.File `json:"files,omitempty" yaml:"-"`
345+
Size int64 `json:"size" yaml:"-"`
346+
SizeUncompressed int64 `json:"size_uncompressed" yaml:"-"`
347+
HSize string `json:"size_h,omitempty" yaml:"size_h,omitempty"`
348+
HSizeUncompressed string `json:"size_uncompressed_h" yaml:"size_uncompressed_h"`
343349
LastWriteTS int64 `json:"last_write_ts" yaml:"-"`
344350
LastTransitionTS int64 `json:"last_transition_ts" yaml:"-"`
345351
LastWriteTime string `json:"last_write_time" yaml:"last_write_time"`
@@ -417,8 +423,13 @@ func describeBackup(
417423
Status: bcp.Status,
418424
Size: bcp.Size,
419425
HSize: byteCountIEC(bcp.Size),
426+
SizeUncompressed: bcp.SizeUncompressed,
427+
HSizeUncompressed: byteCountIEC(bcp.SizeUncompressed),
420428
StorageName: bcp.Store.Name,
421429
}
430+
if bcp.SizeUncompressed > 0 {
431+
rv.HSizeUncompressed = byteCountIEC(bcp.SizeUncompressed)
432+
}
422433
if bcp.Err != "" {
423434
rv.Err = &bcp.Err
424435
}
@@ -442,6 +453,8 @@ func describeBackup(
442453
IsConfigSvr: r.IsConfigSvr,
443454
IsConfigShard: r.IsConfigShard,
444455
Status: r.Status,
456+
Size: r.Size,
457+
SizeUncompressed: r.SizeUncompressed,
445458
LastWriteTS: int64(r.LastWriteTS.T),
446459
LastTransitionTS: r.LastTransitionTS,
447460
LastWriteTime: time.Unix(int64(r.LastWriteTS.T), 0).UTC().Format(time.RFC3339),
@@ -457,6 +470,12 @@ func describeBackup(
457470
if bcp.Type == defs.ExternalBackup {
458471
rv.Replsets[i].Files = r.Files
459472
}
473+
if r.Size > 0 {
474+
rv.Replsets[i].HSize = byteCountIEC(r.Size)
475+
}
476+
if r.SizeUncompressed > 0 {
477+
rv.Replsets[i].HSizeUncompressed = byteCountIEC(r.SizeUncompressed)
478+
}
460479

461480
if !b.coll || bcp.Type != defs.LogicalBackup {
462481
continue

‎cmd/pbm/main.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -752,6 +752,14 @@ func (app *pbmApp) buildRestoreCmd() *cobra.Command {
752752
if len(args) == 1 {
753753
restoreOptions.bcp = args[0]
754754
}
755+
if cmd.Flags().Changed("fallback-enabled") {
756+
val, _ := cmd.Flags().GetBool("fallback-enabled")
757+
restoreOptions.fallback = &val
758+
}
759+
if cmd.Flags().Changed("allow-partly-done") {
760+
val, _ := cmd.Flags().GetBool("allow-partly-done")
761+
restoreOptions.allowPartlyDone = &val
762+
}
755763
return runRestore(app.ctx, app.conn, app.pbm, &restoreOptions, app.node, app.pbmOutF)
756764
}),
757765
}
@@ -808,6 +816,15 @@ func (app *pbmApp) buildRestoreCmd() *cobra.Command {
808816
&restoreOptions.ts, "ts", "",
809817
"MongoDB cluster time to restore to. In <T,I> format (e.g. 1682093090,9). External backups only!",
810818
)
819+
restoreCmd.Flags().Bool(
820+
"fallback-enabled", false, "Enables/disables fallback strategy when doing physical restore.",
821+
)
822+
restoreCmd.Flags().Bool(
823+
"allow-partly-done", false,
824+
"Allows parly done state of the cluster after physical restore. "+
825+
"If enabled (default), partly-done status for RS will be treated as successful restore."+
826+
"If disabled, fallback will be applied when cluster is partly-done.",
827+
)
811828

812829
restoreCmd.Flags().StringVar(&restoreOptions.rsMap, RSMappingFlag, "", RSMappingDoc)
813830
_ = viper.BindPFlag(RSMappingFlag, restoreCmd.Flags().Lookup(RSMappingFlag))

‎cmd/pbm/restore.go

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -39,19 +39,21 @@ var (
3939
)
4040

4141
type restoreOpts struct {
42-
bcp string
43-
pitr string
44-
pitrBase string
45-
wait bool
46-
waitTime time.Duration
47-
extern bool
48-
ns string
49-
nsFrom string
50-
nsTo string
51-
usersAndRoles bool
52-
rsMap string
53-
conf string
54-
ts string
42+
bcp string
43+
pitr string
44+
pitrBase string
45+
wait bool
46+
waitTime time.Duration
47+
extern bool
48+
ns string
49+
nsFrom string
50+
nsTo string
51+
usersAndRoles bool
52+
rsMap string
53+
conf string
54+
ts string
55+
fallback *bool
56+
allowPartlyDone *bool
5557

5658
numParallelColls int32
5759
numInsertionWorkers int32
@@ -139,6 +141,9 @@ func runRestore(
139141
if err := validateRestoreUsersAndRoles(o.usersAndRoles, nss); err != nil {
140142
return nil, errors.Wrap(err, "parse --with-users-and-roles option")
141143
}
144+
if err := validateFallbackOpts(o); err != nil {
145+
return nil, err
146+
}
142147

143148
rsMap, err := parseRSNamesMapping(o.rsMap)
144149
if err != nil {
@@ -415,6 +420,8 @@ func doRestore(
415420
UsersAndRoles: o.usersAndRoles,
416421
RSMap: rsMapping,
417422
External: o.extern,
423+
Fallback: o.fallback,
424+
AllowPartlyDone: o.allowPartlyDone,
418425
},
419426
}
420427
if o.pitr != "" {
@@ -812,6 +819,15 @@ func validateNSFromNSTo(o *restoreOpts) error {
812819
return nil
813820
}
814821

822+
func validateFallbackOpts(o *restoreOpts) error {
823+
if o.fallback != nil && !*o.fallback &&
824+
o.allowPartlyDone != nil && !*o.allowPartlyDone {
825+
return errors.New("It's not possible to disable both --allow-partly-done " +
826+
"and --fallback-enabled at the same time.")
827+
}
828+
return nil
829+
}
830+
815831
func parseCLINumInsertionWorkersOption(value int32) (*int32, error) {
816832
if value < 0 {
817833
return nil, errors.New("Number of insertion workers has to be greater than zero.")

‎pbm/backup/logical.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ func (b *Backup) doLogical(
236236
}
237237
}
238238

239-
err = IncBackupSize(ctx, b.leadConn, bcp.Name, snapshotSize+oplogSize)
239+
err = IncBackupSize(ctx, b.leadConn, bcp.Name, snapshotSize+oplogSize, nil)
240240
if err != nil {
241241
return errors.Wrap(err, "inc backup size")
242242
}

‎pbm/backup/physical.go

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -515,8 +515,10 @@ func (b *Backup) uploadPhysical(
515515
filelist = append(filelist, ju...)
516516

517517
size := int64(0)
518+
sizeUncompressed := int64(0)
518519
for _, f := range filelist {
519520
size += f.StgSize
521+
sizeUncompressed += f.Size
520522
}
521523

522524
filelistPath := path.Join(bcp.Name, rsMeta.Name, FilelistName)
@@ -526,10 +528,29 @@ func (b *Backup) uploadPhysical(
526528
}
527529
l.Info("uploaded: %q %s", filelistPath, storage.PrettySize(flSize))
528530

529-
err = IncBackupSize(ctx, b.leadConn, bcp.Name, size+flSize)
531+
totalSize := size + flSize
532+
totalUncompressed := sizeUncompressed + flSize
533+
err = IncBackupSize(
534+
ctx,
535+
b.leadConn,
536+
bcp.Name,
537+
totalSize,
538+
&totalUncompressed,
539+
)
530540
if err != nil {
531541
return errors.Wrap(err, "inc backup size")
532542
}
543+
err = SetBackupSizeForRS(
544+
ctx,
545+
b.leadConn,
546+
bcp.Name,
547+
rsMeta.Name,
548+
totalSize,
549+
totalUncompressed,
550+
)
551+
if err != nil {
552+
return errors.Wrap(err, "set RS backup size")
553+
}
533554

534555
return nil
535556
}

‎pbm/backup/query.go

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,14 +219,52 @@ func ChangeRSState(conn connect.Client, bcpName, rsName string, s defs.Status, m
219219
return err
220220
}
221221

222-
func IncBackupSize(ctx context.Context, conn connect.Client, bcpName string, size int64) error {
222+
// IncBackupSize increments total backup size.
223+
func IncBackupSize(
224+
ctx context.Context,
225+
conn connect.Client,
226+
bcpName string,
227+
size int64,
228+
sizeUncompressed *int64,
229+
) error {
230+
update := bson.D{
231+
{"$inc", bson.M{"size": size}},
232+
}
233+
if sizeUncompressed != nil {
234+
update = append(
235+
update,
236+
bson.E{"$inc", bson.M{"size_uncompressed": sizeUncompressed}},
237+
)
238+
}
239+
223240
_, err := conn.BcpCollection().UpdateOne(ctx,
224241
bson.D{{"name", bcpName}},
225-
bson.D{{"$inc", bson.M{"size": size}}})
242+
update,
243+
)
226244

227245
return err
228246
}
229247

248+
// SetBackupSizeForRS sets size of backup for specified RS.
249+
func SetBackupSizeForRS(
250+
ctx context.Context,
251+
conn connect.Client,
252+
bcpName,
253+
rsName string,
254+
size int64,
255+
sizeUncompressed int64,
256+
) error {
257+
_, err := conn.BcpCollection().UpdateOne(
258+
ctx,
259+
bson.D{{"name", bcpName}, {"replsets.name", rsName}},
260+
bson.D{
261+
{"$set", bson.M{"replsets.$.size": size}},
262+
{"$set", bson.M{"replsets.$.size_uncompressed": sizeUncompressed}},
263+
},
264+
)
265+
return err
266+
}
267+
230268
func SetRSLastWrite(conn connect.Client, bcpName, rsName string, ts primitive.Timestamp) error {
231269
_, err := conn.BcpCollection().UpdateOne(
232270
context.Background(),

‎pbm/backup/types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ type BackupMeta struct {
4444
Compression compress.CompressionType `bson:"compression" json:"compression"`
4545
Store Storage `bson:"store" json:"store"`
4646
Size int64 `bson:"size" json:"size"`
47+
SizeUncompressed int64 `bson:"size_uncompressed" json:"size_uncompressed"`
4748
MongoVersion string `bson:"mongodb_version" json:"mongodb_version"`
4849
FCV string `bson:"fcv" json:"fcv"`
4950
StartTS int64 `bson:"start_ts" json:"start_ts"`
@@ -117,6 +118,8 @@ type BackupReplset struct {
117118
OplogName string `bson:"oplog_name,omitempty" json:"oplog_name,omitempty"`
118119
StartTS int64 `bson:"start_ts" json:"start_ts"`
119120
Status defs.Status `bson:"status" json:"status"`
121+
Size int64 `bson:"size" json:"size"`
122+
SizeUncompressed int64 `bson:"size_uncompressed" json:"size_uncompressed"`
120123
IsConfigSvr *bool `bson:"iscs,omitempty" json:"iscs,omitempty"`
121124
IsConfigShard *bool `bson:"configshard,omitempty" json:"configshard,omitempty"`
122125
LastTransitionTS int64 `bson:"last_transition_ts" json:"last_transition_ts"`

‎pbm/config/config.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,9 @@ type RestoreConf struct {
370370
// physical restore. Will try $PATH/mongod if not set.
371371
MongodLocation string `bson:"mongodLocation" json:"mongodLocation,omitempty" yaml:"mongodLocation,omitempty"`
372372
MongodLocationMap map[string]string `bson:"mongodLocationMap" json:"mongodLocationMap,omitempty" yaml:"mongodLocationMap,omitempty"`
373+
374+
FallbackEnabled *bool `bson:"fallbackEnabled,omitempty" json:"fallbackEnabled,omitempty" yaml:"fallbackEnabled,omitempty"`
375+
AllowPartlyDone *bool `bson:"allowPartlyDone,omitempty" json:"allowPartlyDone,omitempty" yaml:"allowPartlyDone,omitempty"`
373376
}
374377

375378
func (cfg *RestoreConf) Clone() *RestoreConf {
@@ -388,6 +391,22 @@ func (cfg *RestoreConf) Clone() *RestoreConf {
388391
return &rv
389392
}
390393

394+
// GetFallbackEnabled gets config's or default value for fallbackEnabled
395+
func (cfg *RestoreConf) GetFallbackEnabled() bool {
396+
if cfg != nil && cfg.FallbackEnabled != nil {
397+
return *cfg.FallbackEnabled
398+
}
399+
return false
400+
}
401+
402+
// GetAllowPartlyDone gets config's or default value for allowPartlyDone
403+
func (cfg *RestoreConf) GetAllowPartlyDone() bool {
404+
if cfg != nil && cfg.AllowPartlyDone != nil {
405+
return *cfg.AllowPartlyDone
406+
}
407+
return true
408+
}
409+
391410
//nolint:lll
392411
type BackupConf struct {
393412
OplogSpanMin float64 `bson:"oplogSpanMin" json:"oplogSpanMin" yaml:"oplogSpanMin"`

‎pbm/ctrl/cmd.go

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -150,13 +150,15 @@ func (b BackupCmd) String() string {
150150
}
151151

152152
type RestoreCmd struct {
153-
Name string `bson:"name"`
154-
BackupName string `bson:"backupName"`
155-
Namespaces []string `bson:"nss,omitempty"`
156-
NamespaceFrom string `bson:"nsFrom,omitempty"`
157-
NamespaceTo string `bson:"nsTo,omitempty"`
158-
UsersAndRoles bool `bson:"usersAndRoles,omitempty"`
159-
RSMap map[string]string `bson:"rsMap,omitempty"`
153+
Name string `bson:"name"`
154+
BackupName string `bson:"backupName"`
155+
Namespaces []string `bson:"nss,omitempty"`
156+
NamespaceFrom string `bson:"nsFrom,omitempty"`
157+
NamespaceTo string `bson:"nsTo,omitempty"`
158+
UsersAndRoles bool `bson:"usersAndRoles,omitempty"`
159+
RSMap map[string]string `bson:"rsMap,omitempty"`
160+
Fallback *bool `bson:"fallbackEnabled"`
161+
AllowPartlyDone *bool `bson:"allowPartlyDone"`
160162

161163
NumParallelColls *int32 `bson:"numParallelColls,omitempty"`
162164
NumInsertionWorkers *int32 `bson:"numInsertionWorkers,omitempty"`

0 commit comments

Comments
 (0)