Skip to content

Commit 71014a6

Browse files
1gtmsayedppqq
1gtm
and
sayedppqq
authored
Fix MongoDB url connection str for external databases (#2138) (#2143)
Signed-off-by: Anisur Rahman <anisur@appscode.com> Signed-off-by: sayedppqq <sayed@appscode.com> Co-authored-by: sayedppqq <sayed@appscode.com>
1 parent 4a1ca43 commit 71014a6

File tree

3 files changed

+174
-35
lines changed

3 files changed

+174
-35
lines changed

‎pkg/backup.go

Lines changed: 61 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -267,11 +267,24 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
267267
return nil, err
268268
}
269269

270-
port, err := appBinding.Port()
271-
if err != nil {
272-
return nil, err
270+
var isSrv bool
271+
port := int32(27017)
272+
if appBinding.Spec.ClientConfig.URL != nil {
273+
isSrv, err = isSrvConnection(*appBinding.Spec.ClientConfig.URL)
274+
if err != nil {
275+
return nil, err
276+
}
273277
}
274278

279+
// Checked for Altlas and DigitalOcean srv format connection string don't give port.
280+
// mongodump not support both --uri and --port.
281+
282+
if !isSrv {
283+
port, err = appBinding.Port()
284+
if err != nil {
285+
return nil, err
286+
}
287+
}
275288
waitForDBReady(hostname, port, opt.waitTimeout)
276289

277290
// unmarshal parameter is the field has value
@@ -318,7 +331,12 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
318331
}
319332
}
320333

334+
var tlsEnable bool
321335
if appBinding.Spec.ClientConfig.CABundle != nil {
336+
tlsEnable = true
337+
}
338+
339+
if tlsEnable {
322340
if tlsSecret == nil {
323341
return nil, errors.Wrap(err, "spec.tlsSecret needs to be set in appbinding for TLS secured database.")
324342
}
@@ -333,8 +351,8 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
333351
}
334352
dumpCreds = []interface{}{
335353
"--ssl",
336-
"--sslCAFile", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName),
337-
"--sslPEMKeyFile", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName),
354+
fmt.Sprintf("--sslCAFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName)),
355+
fmt.Sprintf("--sslPEMKeyFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName)),
338356
}
339357

340358
// get certificate secret to get client certificate
@@ -361,9 +379,9 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
361379
return nil, errors.Wrap(err, "unable to get user from ssl.")
362380
}
363381
userAuth := []interface{}{
364-
"-u", user,
365-
"--authenticationMechanism", "MONGODB-X509",
366-
"--authenticationDatabase", "$external",
382+
fmt.Sprintf("--username=%s", user),
383+
"--authenticationMechanism=MONGODB-X509",
384+
"--authenticationDatabase=$external",
367385
}
368386
mongoCreds = append(mongoCreds, userAuth...)
369387
dumpCreds = append(dumpCreds, userAuth...)
@@ -372,7 +390,7 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
372390
userAuth := []interface{}{
373391
fmt.Sprintf("--username=%s", authSecret.Data[MongoUserKey]),
374392
fmt.Sprintf("--password=%s", authSecret.Data[MongoPasswordKey]),
375-
"--authenticationDatabase", opt.authenticationDatabase,
393+
fmt.Sprintf("--authenticationDatabase=%s", opt.authenticationDatabase),
376394
}
377395
mongoCreds = append(mongoCreds, userAuth...)
378396
dumpCreds = append(dumpCreds, userAuth...)
@@ -387,19 +405,32 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
387405
BackupPaths: opt.defaultBackupOptions.BackupPaths,
388406
}
389407

408+
uri := opt.buildMongoURI(mongoDSN, port, isStandalone, isSrv, tlsEnable)
409+
390410
// setup pipe command
391411
backupCmd := restic.Command{
392412
Name: MongoDumpCMD,
393-
Args: append([]interface{}{
394-
"--host", mongoDSN,
413+
Args: []interface{}{
414+
"--uri", fmt.Sprintf("\"%s\"", uri),
395415
"--archive",
396-
}, dumpCreds...),
416+
},
397417
}
398-
userArgs := strings.Fields(opt.mongoArgs)
399418

400-
if isStandalone {
401-
backupCmd.Args = append(backupCmd.Args, fmt.Sprintf("--port=%d", port))
402-
} else {
419+
if tlsEnable {
420+
backupCmd.Args = append(backupCmd.Args,
421+
fmt.Sprintf("--sslCAFile=%s", getOptionValue(dumpCreds, "--sslCAFile")),
422+
fmt.Sprintf("--sslPEMKeyFile=%s", getOptionValue(dumpCreds, "--sslPEMKeyFile")))
423+
}
424+
425+
var userArgs []string
426+
for _, arg := range strings.Fields(opt.mongoArgs) {
427+
// illegal argument combination: cannot specify --db and --uri
428+
if !strings.Contains(arg, "--db") {
429+
userArgs = append(userArgs, arg)
430+
}
431+
}
432+
433+
if !isStandalone {
403434
// - port is already added in mongoDSN with replicasetName/host:port format.
404435
// - oplog is enabled automatically for replicasets.
405436
// Don't use --oplog if user specify any of these arguments through opt.mongoArgs
@@ -558,6 +589,20 @@ func cleanup() {
558589
}
559590
}
560591

592+
func getOptionValue(args []interface{}, option string) string {
593+
for _, arg := range args {
594+
strArg, ok := arg.(string)
595+
if !ok {
596+
continue
597+
}
598+
// assuming value has '='
599+
if strings.HasPrefix(strArg, option+"=") {
600+
return strings.TrimPrefix(strArg, option+"=")
601+
}
602+
}
603+
return ""
604+
}
605+
561606
func (opt *mongoOptions) getHostBackupStats(err error) []api_v1beta1.HostBackupStats {
562607
var backupStats []api_v1beta1.HostBackupStats
563608

‎pkg/restore.go

Lines changed: 50 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -204,9 +204,23 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
204204
return nil, err
205205
}
206206

207-
port, err := appBinding.Port()
208-
if err != nil {
209-
return nil, err
207+
var isSrv bool
208+
port := int32(27017)
209+
if appBinding.Spec.ClientConfig.URL != nil {
210+
isSrv, err = isSrvConnection(*appBinding.Spec.ClientConfig.URL)
211+
if err != nil {
212+
return nil, err
213+
}
214+
}
215+
216+
// Checked for Altlas and DigitalOcean srv format connection string don't give port.
217+
// mongodump --uri format not support port.
218+
219+
if !isSrv {
220+
port, err = appBinding.Port()
221+
if err != nil {
222+
return nil, err
223+
}
210224
}
211225

212226
// unmarshal parameter is the field has value
@@ -249,8 +263,12 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
249263
return nil, err
250264
}
251265
}
252-
266+
var tlsEnable bool
253267
if appBinding.Spec.ClientConfig.CABundle != nil {
268+
tlsEnable = true
269+
}
270+
271+
if tlsEnable {
254272
if err := os.WriteFile(filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil {
255273
return nil, errors.Wrap(err, "failed to write key for CA certificate")
256274
}
@@ -261,8 +279,8 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
261279
}
262280
dumpCreds = []interface{}{
263281
"--ssl",
264-
"--sslCAFile", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName),
265-
"--sslPEMKeyFile", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName),
282+
fmt.Sprintf("--sslCAFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName)),
283+
fmt.Sprintf("--sslPEMKeyFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName)),
266284
}
267285

268286
// get certificate secret to get client certificate
@@ -289,9 +307,9 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
289307
return nil, errors.Wrap(err, "unable to get user from ssl.")
290308
}
291309
userAuth := []interface{}{
292-
"-u", user,
293-
"--authenticationMechanism", "MONGODB-X509",
294-
"--authenticationDatabase", "$external",
310+
fmt.Sprintf("--username=%s", user),
311+
"--authenticationMechanism=MONGODB-X509",
312+
"--authenticationDatabase=$external",
295313
}
296314
mongoCreds = append(mongoCreds, userAuth...)
297315
dumpCreds = append(dumpCreds, userAuth...)
@@ -300,7 +318,7 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
300318
userAuth := []interface{}{
301319
fmt.Sprintf("--username=%s", authSecret.Data[MongoUserKey]),
302320
fmt.Sprintf("--password=%s", authSecret.Data[MongoPasswordKey]),
303-
"--authenticationDatabase", opt.authenticationDatabase,
321+
fmt.Sprintf("--authenticationDatabase=%s", opt.authenticationDatabase),
304322
}
305323
mongoCreds = append(mongoCreds, userAuth...)
306324
dumpCreds = append(dumpCreds, userAuth...)
@@ -314,19 +332,32 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
314332
FileName: opt.defaultDumpOptions.FileName,
315333
Snapshot: opt.getSnapshotForHost(hostKey, restoreSession.Spec.Target.Rules),
316334
}
335+
336+
uri := opt.buildMongoURI(mongoDSN, port, isStandalone, isSrv, tlsEnable)
337+
317338
// setup pipe command
318339
restoreCmd := restic.Command{
319340
Name: MongoRestoreCMD,
320-
Args: append([]interface{}{
321-
"--host", mongoDSN,
341+
Args: []interface{}{
342+
"--uri", fmt.Sprintf("\"%s\"", uri),
322343
"--archive",
323-
}, dumpCreds...),
344+
},
345+
}
346+
if tlsEnable {
347+
restoreCmd.Args = append(restoreCmd.Args,
348+
fmt.Sprintf("--sslCAFile=%s", getOptionValue(dumpCreds, "--sslCAFile")),
349+
fmt.Sprintf("--sslPEMKeyFile=%s", getOptionValue(dumpCreds, "--sslPEMKeyFile")))
350+
}
351+
352+
var userArgs []string
353+
for _, arg := range strings.Fields(opt.mongoArgs) {
354+
// illegal argument combination: cannot specify --db and --uri
355+
if !strings.Contains(arg, "--db") {
356+
userArgs = append(userArgs, arg)
357+
}
324358
}
325359

326-
userArgs := strings.Fields(opt.mongoArgs)
327-
if isStandalone {
328-
restoreCmd.Args = append(restoreCmd.Args, fmt.Sprintf("--port=%d", port))
329-
} else {
360+
if !isStandalone {
330361
// - port is already added in mongoDSN with replicasetName/host:port format.
331362
// - oplog is enabled automatically for replicasets.
332363
// Don't use --oplogReplay if user specify any of these arguments through opt.mongoArgs
@@ -369,11 +400,11 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
369400
// ref: https://docs.mongodb.com/manual/tutorial/backup-sharded-cluster-with-database-dumps/
370401

371402
if parameters.ConfigServer != "" {
372-
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(parameters.ConfigServer, MongoConfigSVRHostKey, false))
403+
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(extractHost(parameters.ConfigServer), MongoConfigSVRHostKey, false))
373404
}
374405

375406
for key, host := range parameters.ReplicaSets {
376-
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(host, key, false))
407+
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(extractHost(host), key, false))
377408
}
378409

379410
// if parameters.ReplicaSets is nil, then perform normal backup with clientconfig.Service.Name mongo dsn

‎pkg/utils.go

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package pkg
1818

1919
import (
2020
"fmt"
21+
"net/url"
2122
"os/exec"
2223
"strings"
2324
"time"
@@ -114,3 +115,65 @@ func getTime(t string) (time.Time, error) {
114115
}
115116
return parsedTime, nil
116117
}
118+
119+
func isSrvConnection(connectionString string) (bool, error) {
120+
parsedURL, err := url.Parse(connectionString)
121+
if err != nil {
122+
return false, err
123+
}
124+
125+
// Check if the scheme is "mongodb+srv"
126+
return parsedURL.Scheme == "mongodb+srv", nil
127+
}
128+
129+
func (opt *mongoOptions) buildMongoURI(mongoDSN string, port int32, isStandalone, isSrv, tlsEnable bool) string {
130+
prefix, ssl := "mongodb", ""
131+
portStr := fmt.Sprintf(":%d", port)
132+
if isSrv {
133+
prefix += "+srv"
134+
}
135+
if !isStandalone || isSrv {
136+
portStr = ""
137+
}
138+
139+
backupDb := getBackupDB(opt.mongoArgs) // "" stands for all databases.
140+
authDbName := getOptionValue(dumpCreds, "--authenticationDatabase")
141+
userName := getOptionValue(dumpCreds, "--username")
142+
password := getOptionValue(dumpCreds, "--password")
143+
authMechanism := getOptionValue(dumpCreds, "--authenticationMechanism")
144+
145+
if password != "" {
146+
password = fmt.Sprintf(":%s", password)
147+
}
148+
if authMechanism == "" {
149+
authMechanism = "SCRAM-SHA-256"
150+
}
151+
if tlsEnable {
152+
ssl = "&ssl=true"
153+
}
154+
155+
return fmt.Sprintf("%s://%s%s@%s%s/%s?authSource=%s&authMechanism=%s%s",
156+
prefix, userName, password, mongoDSN, portStr, backupDb, authDbName, authMechanism, ssl)
157+
}
158+
159+
// remove "shard0/" prefix from shard0/simple-shard0-0.simple-shard0-pods.demo.svc:27017,simple-shard0-1.simple-shard0-pods.demo.svc:27017
160+
func extractHost(host string) string {
161+
index := strings.Index(host, "/")
162+
if index != -1 {
163+
host = host[index+1:]
164+
}
165+
return host
166+
}
167+
168+
func getBackupDB(mongoArgs string) string {
169+
backupdb := "" // full
170+
if strings.Contains(mongoArgs, "--db") {
171+
args := strings.Fields(mongoArgs)
172+
for _, arg := range args {
173+
if strings.Contains(arg, "--db") {
174+
backupdb = strings.Split(arg, "=")[1]
175+
}
176+
}
177+
}
178+
return backupdb
179+
}

0 commit comments

Comments
 (0)