Skip to content

Commit

Permalink
Add support for backup and restore from different namespace (#1354) (#…
Browse files Browse the repository at this point in the history 
…1358)

/cherry-pick
Signed-off-by: hmsayem hmsayem@appscode.com

Co-authored-by: Hossain Mahmud <hmsayem@appscode.com>
  • Loading branch information
@1gtm @hmsayem
1gtm and hmsayem committed Jan 20, 2022
1 parent 46bb924 commit 2a31584
Show file tree
Hide file tree
Showing 127 changed files with 27,232 additions and 11,232 deletions.
7 changes: 7 additions & 0 deletions Makefile
View file
Expand Up @@ -362,3 +362,10 @@ release:
.PHONY: clean
clean:
rm -rf .go bin

# make and load docker image to kind cluster
.PHONY: push-to-kind
push-to-kind: container
@echo "Loading docker image into kind cluster...."
@kind load docker-image $(REGISTRY)/stash-mongodb:$(TAG)
@echo "Image has been pushed successfully into kind cluster."
5 changes: 3 additions & 2 deletions go.mod
View file
Expand Up @@ -15,10 +15,11 @@ require (
k8s.io/apimachinery v0.21.1
k8s.io/client-go v0.21.1
k8s.io/klog/v2 v2.8.0
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd
kmodules.xyz/offshoot-api v0.0.0-20211103060642-3e217667cf41
kubedb.dev/apimachinery v0.23.0
stash.appscode.dev/apimachinery v0.17.0
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3
)

replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d
Expand Down
20 changes: 11 additions & 9 deletions go.sum
View file
Expand Up @@ -605,9 +605,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY=
github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
Expand Down Expand Up @@ -928,8 +926,9 @@ golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5f
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down Expand Up @@ -1104,6 +1103,7 @@ k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRV
k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
k8s.io/code-generator v0.21.0/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw=
k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
k8s.io/component-helpers v0.21.0/go.mod h1:tezqefP7lxfvJyR+0a+6QtVrkZ/wIkyMLK4WcQ3Cj8U=
k8s.io/component-helpers v0.21.1/go.mod h1:FtC1flbiQlosHQrLrRUulnKxE4ajgWCGy/67fT2GRlQ=
Expand Down Expand Up @@ -1147,13 +1147,14 @@ kmodules.xyz/client-go v0.0.0-20210928133955-8df5bb467db6/go.mod h1:0gkPeALtYjB2
kmodules.xyz/client-go v0.0.0-20211013093146-1fbfd52e78c9/go.mod h1:0gkPeALtYjB27OHt4rd6+ZmMgoVTHVLtEJQeU23/gtA=
kmodules.xyz/client-go v0.0.0-20211107190155-5bb4090d2728/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I=
kmodules.xyz/client-go v0.0.0-20211110065417-091bd089a92d/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I=
kmodules.xyz/client-go v0.0.0-20211122091731-6c471b24a4ea h1:z5Li57oxum0018ryWpI5w5HYVFgI2S2cVj27R76IRnU=
kmodules.xyz/client-go v0.0.0-20211122091731-6c471b24a4ea/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I=
kmodules.xyz/client-go v0.0.0-20220104114408-2a3a05dbe89f/go.mod h1:xxl1ve1Obe4xaW+XjXsNHyLTni4QPIvHn9TfnYEoQRo=
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe h1:EZE/eC9UF5/wcdHiinzQDVshOceIP9uRxT7qM4BnUQg=
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe/go.mod h1:xxl1ve1Obe4xaW+XjXsNHyLTni4QPIvHn9TfnYEoQRo=
kmodules.xyz/constants v0.0.0-20210218100002-2c304bfda278/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY=
kmodules.xyz/crd-schema-fuzz v0.0.0-20210618002152-fae23aef5fb4/go.mod h1:IIkUctlfoptoci0BOrsUf8ya+MOG5uaeh1PE4uzaIbA=
kmodules.xyz/custom-resources v0.0.0-20211007080833-72bd9e8cae6e/go.mod h1:pGabego8q4oi/2sNjhdtFkgVaVw4AyGv14GO6VtAjTw=
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52 h1:UWVpU7y5znTUusU+JhPB+ojh26f6K2v8sNb37U1DolQ=
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52/go.mod h1:yHLFe4wVYxepTnN00CFUf29xH+jEHDokq6d2fbp9pks=
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd h1:EMcK5eA42CO9Cn290VOy2WIt2YSEbQItlMq8lU+831M=
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd/go.mod h1:/XjDeILFV2wBota5kHo21DMvOt08nSAk1vm6buCuwt4=
kmodules.xyz/monitoring-agent-api v0.0.0-20210928135619-38ca075a2dbd/go.mod h1:08pBqfEuy29EjhaMrHB2XFy2iekoFi7AjaXcJS+xAck=
kmodules.xyz/objectstore-api v0.0.0-20210928135706-fdf68f88ea6e/go.mod h1:Tkcf9uTplnrJ6C8o0zlw2kpgS1SaWAiMO5P2YgLjTo8=
kmodules.xyz/objectstore-api v0.0.0-20211116180107-8720be0c9bf7 h1:JIAEFjN3GDhLEG1Fh1zYpy/QFyyN337mJTM+ODEGosg=
Expand All @@ -1167,6 +1168,7 @@ kmodules.xyz/prober v0.0.0-20210618020259-5836fb959027/go.mod h1:H4NcvS1RQxeXtQO
kmodules.xyz/resource-metadata v0.6.4/go.mod h1:KWf68Ado/hgYpb/msYNvhYSLWvS/bJcVAAHO1/q9nNg=
kmodules.xyz/resource-metrics v0.0.3/go.mod h1:6Dv63HDgp83DhA+lZNB7GIQR6PLjNrYW6ghQKioQzII=
kmodules.xyz/resource-metrics v0.0.5/go.mod h1:6Dv63HDgp83DhA+lZNB7GIQR6PLjNrYW6ghQKioQzII=
kmodules.xyz/resource-metrics v0.0.6/go.mod h1:M7rWuo2qh3BpHhogiEVPnvGY9Xx4Pfygqn1Rex8YbgM=
kmodules.xyz/webhook-runtime v0.0.0-20210928141616-7f73c2ab318a/go.mod h1:MFZFmJk9IXNHwq8JlF/mukwBDbopFQj4swaB2MWHc/U=
kubedb.dev/apimachinery v0.23.0 h1:K0dKXx7XJINv3Py75D/up6V3zl8XfX/e/rvspNejXNA=
kubedb.dev/apimachinery v0.23.0/go.mod h1:x8UBaJPIBCD6S58VQ+38+QxFUXCYdvFrmP9FnOuPOaI=
Expand Down Expand Up @@ -1204,5 +1206,5 @@ software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237/go.mod h1:
software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
stash.appscode.dev/apimachinery v0.16.0/go.mod h1:unpV/YyHVECNrAdFjFg/SetmPeUjfvc2+P8cnGjih/U=
stash.appscode.dev/apimachinery v0.17.0 h1:v0razjOko1K0npOwRzvtymdMByL/cWSNQBgaIq7KaoU=
stash.appscode.dev/apimachinery v0.17.0/go.mod h1:hmxBy6Ei6RjBLgXw4A1hE4eyEgsa43H2LCs1yDfI3GM=
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3 h1:jWOAWMKDc15vVPqq4pToYyRkitlGh6Y2+1V5QgGZsYY=
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3/go.mod h1:BGM/ztGKtXZrtz/voZzRqop8KbbZ+pFI4YeSLVb2MB0=
57 changes: 34 additions & 23 deletions pkg/backup.go
View file
Expand Up @@ -100,10 +100,10 @@ func NewCmdBackup() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error {
defer cleanup()

flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "secret-dir")
flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "storage-secret-name", "storage-secret-namespace")

// catch sigkill signals and gracefully terminate so that cleanup functions are executed.
sigChan := make(chan os.Signal)
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
go func() {
rcvSig := <-sigChan
Expand Down Expand Up @@ -179,11 +179,13 @@ func NewCmdBackup() *cobra.Command {
cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend")
cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored")
cmd.Flags().StringVar(&opt.setupOptions.SecretDir, "secret-dir", opt.setupOptions.SecretDir, "Directory where storage secret has been mounted")
cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory")
cmd.Flags().BoolVar(&opt.setupOptions.EnableCache, "enable-cache", opt.setupOptions.EnableCache, "Specify whether to enable caching for restic")
cmd.Flags().Int64Var(&opt.setupOptions.MaxConnections, "max-connections", opt.setupOptions.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")

cmd.Flags().StringVar(&opt.storageSecret.Name, "storage-secret-name", opt.storageSecret.Name, "Name of the storage secret")
cmd.Flags().StringVar(&opt.storageSecret.Namespace, "storage-secret-namespace", opt.storageSecret.Namespace, "Namespace of the storage secret")
cmd.Flags().StringVar(&opt.authenticationDatabase, "authentication-database", "admin", "Specify the authentication database")
cmd.Flags().StringVar(&opt.defaultBackupOptions.Host, "hostname", opt.defaultBackupOptions.Host, "Name of the host machine")

cmd.Flags().Int64Var(&opt.defaultBackupOptions.RetentionPolicy.KeepLast, "retention-keep-last", opt.defaultBackupOptions.RetentionPolicy.KeepLast, "Specify value for retention strategy")
Expand All @@ -202,6 +204,12 @@ func NewCmdBackup() *cobra.Command {
}

func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic.BackupOutput, error) {
var err error
opt.setupOptions.StorageSecret, err = opt.kubeClient.CoreV1().Secrets(opt.storageSecret.Namespace).Get(context.TODO(), opt.storageSecret.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}

// if any pre-backup actions has been assigned to it, execute them
actionOptions := api_util.ActionOptions{
StashClient: opt.stashClient,
Expand All @@ -210,7 +218,7 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
BackupSessionName: opt.backupSessionName,
Namespace: opt.namespace,
}
err := api_util.ExecutePreBackupActions(actionOptions)
err = api_util.ExecutePreBackupActions(actionOptions)
if err != nil {
return nil, err
}
Expand All @@ -229,25 +237,32 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
return nil, err
}

// get app binding
appBinding, err := opt.catalogClient.AppcatalogV1alpha1().AppBindings(opt.namespace).Get(context.TODO(), opt.appBindingName, metav1.GetOptions{})
if err != nil {
return nil, err
}
// get secret

appBindingSecret, err := opt.kubeClient.CoreV1().Secrets(opt.namespace).Get(context.TODO(), appBinding.Spec.Secret.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}

// transform secret
err = appBinding.TransformSecret(opt.kubeClient, appBindingSecret.Data)
if err != nil {
return nil, err
}

// wait for DB ready
waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port, opt.waitTimeout)
hostname, err := appBinding.Hostname()
if err != nil {
return nil, err
}

port, err := appBinding.Port()
if err != nil {
return nil, err
}

waitForDBReady(hostname, port, opt.waitTimeout)

// unmarshal parameter is the field has value
parameters := v1alpha1.MongoDBConfiguration{}
Expand Down Expand Up @@ -335,7 +350,7 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
adminCreds = []interface{}{
"--username", string(appBindingSecret.Data[MongoUserKey]),
"--password", string(appBindingSecret.Data[MongoPasswordKey]),
"--authenticationDatabase", "admin",
"--authenticationDatabase", opt.authenticationDatabase,
}
}

Expand All @@ -359,7 +374,7 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
userArgs := strings.Fields(opt.mongoArgs)

if isStandalone {
backupCmd.Args = append(backupCmd.Args, "--port="+fmt.Sprint(appBinding.Spec.ClientConfig.Service.Port))
backupCmd.Args = append(backupCmd.Args, fmt.Sprintf("--port=%d", port))
} else {
// - port is already added in mongoDSN with replicasetName/host:port format.
// - oplog is enabled automatically for replicasets.
Expand Down Expand Up @@ -404,17 +419,16 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic

if parameters.ConfigServer != "" {
// sharded cluster. so disable the balancer first. then perform the 'usual' tasks.

primary, secondary, err := getPrimaryNSecondaryMember(parameters.ConfigServer)
if err != nil {
return nil, err
}

// connect to mongos to disable/enable balancer
err = disabelBalancer(appBinding.Spec.ClientConfig.Service.Name)
err = disabelBalancer(hostname)
cleanupFuncs = append(cleanupFuncs, func() error {
// even if error occurs, try to enable the balancer on exiting the program.
return enableBalancer(appBinding.Spec.ClientConfig.Service.Name)
return enableBalancer(hostname)
})
if err != nil {
return nil, err
Expand All @@ -428,14 +442,15 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
}

err = lockConfigServer(parameters.ConfigServer, secondary)

cleanupFuncs = append(cleanupFuncs, func() error {
// even if error occurs, try to unlock the server
return unlockSecondaryMember(secondary)
})
if err != nil {
klog.Errorf("error while locking config server. error: %v", err)
return nil, err
}

opt.backupOptions = append(opt.backupOptions, getBackupOpt(backupHost, MongoConfigSVRHostKey, false))
}

Expand Down Expand Up @@ -470,7 +485,7 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
// if parameters.ReplicaSets is nil, then the mongodb database doesn't have replicasets or sharded replicasets.
// In this case, perform normal backup with clientconfig.Service.Name mongo dsn
if parameters.ReplicaSets == nil {
opt.backupOptions = append(opt.backupOptions, getBackupOpt(appBinding.Spec.ClientConfig.Service.Name, restic.DefaultHost, true))
opt.backupOptions = append(opt.backupOptions, getBackupOpt(hostname, restic.DefaultHost, true))
}

klog.Infoln("processing backup.")
Expand All @@ -482,7 +497,6 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
// hide password, don't print cmd
resticWrapper.HideCMD()

// Run backup
return resticWrapper.RunParallelBackup(opt.backupOptions, targetRef, opt.maxConcurrency)
}

Expand Down Expand Up @@ -604,12 +618,12 @@ func enableBalancer(mongosHost string) error {

func lockConfigServer(configSVRDSN, secondaryHost string) error {
klog.Infoln("Attempting to lock configserver", configSVRDSN)

if secondaryHost == "" {
klog.Warningln("locking configserver is skipped. secondary host is empty")
return nil
}
v := make(map[string]interface{})

// findAndModify BackupControlDocument. skip single quote inside single quote: https://stackoverflow.com/a/28786747/4628962
args := append([]interface{}{
"config",
Expand All @@ -620,12 +634,10 @@ func lockConfigServer(configSVRDSN, secondaryHost string) error {
if err := sh.Command(MongoCMD, args...).Command("tail", "-1").UnmarshalJSON(&v); err != nil {
return err
}

val, ok := v["counter"].(float64)
if !ok || int(val) == 0 {
return fmt.Errorf("unable to modify BackupControlDocument. got response: %v", v)
}

val2 := float64(0)
timer := 0 // wait approximately 5 minutes.
for timer < 60 && (int(val2) == 0 || int(val) != int(val2)) {
Expand All @@ -635,8 +647,9 @@ func lockConfigServer(configSVRDSN, secondaryHost string) error {
"config",
"--host", secondaryHost,
"--quiet",
"--eval", "rs.slaveOk(); db.BackupControl.find({ '_id' : 'BackupControlDocument' }).readConcern('majority');",
"--eval", "rs.secondaryOk(); db.BackupControl.find({ '_id' : 'BackupControlDocument' }).readConcern('majority');",
}, adminCreds...)

if err := sh.Command(MongoCMD, args...).UnmarshalJSON(&v); err != nil {
return err
}
Expand All @@ -645,7 +658,6 @@ func lockConfigServer(configSVRDSN, secondaryHost string) error {
if !ok {
return fmt.Errorf("unable to get BackupControlDocument. got response: %v", v)
}

if int(val) != int(val2) {
klog.V(5).Infof("BackupDocument counter in secondary is not same. Expected %v, but got %v. Full response: %v", val, val2, v)
time.Sleep(time.Second * 5)
Expand All @@ -654,7 +666,6 @@ func lockConfigServer(configSVRDSN, secondaryHost string) error {
if timer >= 60 {
return fmt.Errorf("timeout while waiting for BackupDocument counter in secondary to be same as primary. Expected %v, but got %v. Full response: %v", val, val2, v)
}

// lock secondary
return lockSecondaryMember(secondaryHost)
}
Expand Down

0 comments on commit 2a31584

Please sign in to comment.