add support for dumping large database test

cmd/dump_test.go

@@ -30,35 +30,35 @@ func TestDumpCmd(t *testing.T) {
 	}{
 		// invalid ones
 		{"missing server and target options", []string{""}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
-		{"invalid target URL", []string{"--server", "abc", "--target", "def"}, "", true, core.DumpOptions{DBConn: database.Connection{Host: "abc"}}, core.TimerOptions{}, nil},
+		{"invalid target URL", []string{"--server", "abc", "--target", "def"}, "", true, core.DumpOptions{DBConn: &database.Connection{Host: "abc"}}, core.TimerOptions{}, nil},
 
 		// file URL
 		{"file URL", []string{"--server", "abc", "--target", "file:///foo/bar"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"file URL with pass-file", []string{"--server", "abc", "--target", "file:///foo/bar", "--pass-file", "testdata/password.txt"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort, Pass: "testpassword"},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort, Pass: "testpassword"},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"file URL with pass and pass-file (pass takes precedence)", []string{"--server", "abc", "--target", "file:///foo/bar", "--pass", "explicitpass", "--pass-file", "testdata/password.txt"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort, Pass: "explicitpass"},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort, Pass: "explicitpass"},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"file URL with prune", []string{"--server", "abc", "--target", "file:///foo/bar", "--retention", "1h"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 
@@ -67,14 +67,14 @@ func TestDumpCmd(t *testing.T) {
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"database explicit name with explicit port", []string{"--server", "abc", "--port", "3307", "--target", "file:///foo/bar"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: 3307},
+			DBConn:           &database.Connection{Host: "abc", Port: 3307},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 
@@ -83,21 +83,21 @@ func TestDumpCmd(t *testing.T) {
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
+			DBConn:           &database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 		{"config file with port override", []string{"--config-file", "testdata/config.yml", "--port", "3307"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abcd", Port: 3307, User: "user2", Pass: "xxxx2"},
+			DBConn:           &database.Connection{Host: "abcd", Port: 3307, User: "user2", Pass: "xxxx2"},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 		{"config file with filename pattern override", []string{"--config-file", "testdata/pattern.yml", "--port", "3307"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abcd", Port: 3307, User: "user2", Pass: "xxxx2"},
+			DBConn:           &database.Connection{Host: "abcd", Port: 3307, User: "user2", Pass: "xxxx2"},
 			FilenamePattern:  "foo_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 
@@ -106,60 +106,60 @@ func TestDumpCmd(t *testing.T) {
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Once: true, Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"cron flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Cron: "0 0 * * *"}, nil},
 		{"begin flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--begin", "1234"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: "1234"}, nil},
 		{"frequency flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--frequency", "10"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort},
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: 10, Begin: defaultBegin}, nil},
 		{"incompatible flags: once/cron", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--cron", "0 0 * * *"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
 		{"incompatible flags: once/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
 		{"incompatible flags: once/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--frequency", "10"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
 		{"incompatible flags: cron/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
 		{"incompatible flags: cron/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--frequency", "10"}, "", true, core.DumpOptions{
-			DBConn: database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
+			DBConn: &database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 
 		// pre- and post-backup scripts
 		{"prebackup scripts", []string{"--server", "abc", "--target", "file:///foo/bar", "--pre-backup-scripts", "/prebackup"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:           &database.Connection{Host: "abc", Port: defaultPort},
 			PreBackupScripts: "/prebackup",
 			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"postbackup scripts", []string{"--server", "abc", "--target", "file:///foo/bar", "--post-backup-scripts", "/postbackup"}, "", false, core.DumpOptions{
 			Targets:           []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket:  defaultMaxAllowedPacket,
 			Compressor:        &compression.GzipCompressor{},
-			DBConn:            database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:            &database.Connection{Host: "abc", Port: defaultPort},
 			PostBackupScripts: "/postbackup",
 			FilenamePattern:   "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"prebackup and postbackup scripts", []string{"--server", "abc", "--target", "file:///foo/bar", "--post-backup-scripts", "/postbackup", "--pre-backup-scripts", "/prebackup"}, "", false, core.DumpOptions{
 			Targets:           []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket:  defaultMaxAllowedPacket,
 			Compressor:        &compression.GzipCompressor{},
-			DBConn:            database.Connection{Host: "abc", Port: defaultPort},
+			DBConn:            &database.Connection{Host: "abc", Port: defaultPort},
 			PreBackupScripts:  "/prebackup",
 			PostBackupScripts: "/postbackup",
 			FilenamePattern:   "db_backup_{{ .now }}.{{ .compression }}",

cmd/restore_test.go

@@ -33,7 +33,7 @@ func TestRestoreCmd(t *testing.T) {
 		{"missing server and target options", []string{""}, "", true, core.RestoreOptions{}},
 		{"invalid target URL", []string{"--server", "abc", "--target", "def"}, "", true, core.RestoreOptions{}},
 		{"valid URL missing dump filename", []string{"--server", "abc", "--target", "file:///foo/bar"}, "", true, core.RestoreOptions{}},
-		{"valid file URL", []string{"--server", "abc", "--target", fileTarget, "filename.tgz", "--verbose", "2"}, "", false, core.RestoreOptions{Target: file.New(*fileTargetURL), TargetFile: "filename.tgz", DBConn: database.Connection{Host: "abc", Port: defaultPort}, DatabasesMap: map[string]string{}, Compressor: &compression.GzipCompressor{}}},
+		{"valid file URL", []string{"--server", "abc", "--target", fileTarget, "filename.tgz", "--verbose", "2"}, "", false, core.RestoreOptions{Target: file.New(*fileTargetURL), TargetFile: "filename.tgz", DBConn: &database.Connection{Host: "abc", Port: defaultPort}, DatabasesMap: map[string]string{}, Compressor: &compression.GzipCompressor{}}},
 	}
 
 	for _, tt := range tests {

cmd/root.go

@@ -38,7 +38,7 @@ type subCommand func(execs, *cmdConfiguration) (*cobra.Command, error)
 var subCommands = []subCommand{dumpCmd, restoreCmd, pruneCmd}
 
 type cmdConfiguration struct {
-	dbconn        database.Connection
+	dbconn        *database.Connection
 	creds         credentials.Creds
 	configuration *api.ConfigSpec
 	logger        *log.Logger
@@ -111,6 +111,11 @@ func rootCmd(execs execs) (*cobra.Command, error) {
 			// the structure of our config file is more complex and with relationships than our config/env var
 			// so we cannot use a single viper structure, as described above.
 
+			// avoid nil pointer dereference
+			if cmdConfig.dbconn == nil {
+				cmdConfig.dbconn = &database.Connection{}
+			}
+
 			// set up database connection
 			if actualConfig != nil {
 				if actualConfig.Database != nil {

pkg/core/dump.go

@@ -111,6 +111,7 @@ func (e *Executor) Dump(ctx context.Context, opts DumpOptions) (DumpResults, err
 		Routines:            routines,
 		SuppressUseDatabase: suppressUseDatabase,
 		MaxAllowedPacket:    maxAllowedPacket,
+		PostDumpDelay:       opts.PostDumpDelay,
 	}, dw); err != nil {
 		dbDumpSpan.SetStatus(codes.Error, err.Error())
 		dbDumpSpan.End()

pkg/core/dumpoptions.go

@@ -1,6 +1,8 @@
 package core
 
 import (
+	"time"
+
 	"github.com/databacker/mysql-backup/pkg/compression"
 	"github.com/databacker/mysql-backup/pkg/database"
 	"github.com/databacker/mysql-backup/pkg/encrypt"
@@ -12,7 +14,7 @@ type DumpOptions struct {
 	Targets             []storage.Storage
 	Safechars           bool
 	DBNames             []string
-	DBConn              database.Connection
+	DBConn              *database.Connection
 	Compressor          compression.Compressor
 	Encryptor           encrypt.Encryptor
 	Exclude             []string
@@ -25,4 +27,6 @@ type DumpOptions struct {
 	MaxAllowedPacket    int
 	Run                 uuid.UUID
 	FilenamePattern     string
+	// PostDumpDelay inafter each dump is complete, while holding connection open. Do not use outside of tests.
+	PostDumpDelay time.Duration
 }

pkg/core/restoreoptions.go

@@ -10,7 +10,7 @@ import (
 type RestoreOptions struct {
 	Target       storage.Storage
 	TargetFile   string
-	DBConn       database.Connection
+	DBConn       *database.Connection
 	DatabasesMap map[string]string
 	Compressor   compression.Compressor
 	Run          uuid.UUID

pkg/database/connection.go

@@ -1,31 +1,48 @@
 package database
 
 import (
+	"database/sql"
 	"fmt"
 	"strings"
 
 	mysql "github.com/go-sql-driver/mysql"
 )
 
 type Connection struct {
-	User string
-	Pass string
-	Host string
-	Port int
+	User            string
+	Pass            string
+	Host            string
+	Port            int
+	MultiStatements bool
+
+	// holds a connection to the database
+	sql *sql.DB
 }
 
-func (c Connection) MySQL() string {
-	config := mysql.NewConfig()
-	config.User = c.User
-	config.Passwd = c.Pass
-	if strings.HasPrefix(c.Host, "/") {
-		config.Net = "unix"
-		config.Addr = c.Host
-	} else {
-		config.Net = "tcp"
-		config.Addr = fmt.Sprintf("%s:%d", c.Host, c.Port)
+// MySQL returns a MySQL connection for the Connection.
+func (c *Connection) MySQL() (*sql.DB, error) {
+	if c.sql == nil {
+
+		config := mysql.NewConfig()
+		config.User = c.User
+		config.Passwd = c.Pass
+		if strings.HasPrefix(c.Host, "/") {
+			config.Net = "unix"
+			config.Addr = c.Host
+		} else {
+			config.Net = "tcp"
+			config.Addr = fmt.Sprintf("%s:%d", c.Host, c.Port)
+		}
+		config.ParseTime = true
+		config.TLSConfig = "preferred"
+		config.MultiStatements = c.MultiStatements
+		dsn := config.FormatDSN()
+		handle, err := sql.Open("mysql", dsn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to open connection to database: %v", err)
+		}
+		c.sql = handle
 	}
-	config.ParseTime = true
-	config.TLSConfig = "preferred"
-	return config.FormatDSN()
+	return c.sql, nil
+
 }

pkg/database/dump.go

@@ -2,8 +2,8 @@ package database
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
+	"time"
 
 	"github.com/databacker/mysql-backup/pkg/database/mysql"
 )
@@ -14,9 +14,11 @@ type DumpOpts struct {
 	Routines            bool
 	SuppressUseDatabase bool
 	MaxAllowedPacket    int
+	// PostDumpDelay after each dump is complete, while holding connection open. Do not use outside of tests.
+	PostDumpDelay time.Duration
 }
 
-func Dump(ctx context.Context, dbconn Connection, opts DumpOpts, writers []DumpWriter) error {
+func Dump(ctx context.Context, dbconn *Connection, opts DumpOpts, writers []DumpWriter) error {
 
 	// TODO: dump data for each writer:
 	// per schema
@@ -25,12 +27,11 @@ func Dump(ctx context.Context, dbconn Connection, opts DumpOpts, writers []DumpW
 	//    mysqldump -A $MYSQLDUMP_OPTS
 	// all at once limited to some databases
 	//    mysqldump --databases $DB_DUMP_INCLUDE $MYSQLDUMP_OPTS
+	db, err := dbconn.MySQL()
+	if err != nil {
+		return fmt.Errorf("failed to open connection to database: %v", err)
+	}
 	for _, writer := range writers {
-		db, err := sql.Open("mysql", dbconn.MySQL())
-		if err != nil {
-			return fmt.Errorf("failed to open connection to database: %v", err)
-		}
-		defer func() { _ = db.Close() }()
 		for _, schema := range writer.Schemas {
 			dumper := &mysql.Data{
 				Out:                 writer.Writer,
@@ -42,6 +43,7 @@ func Dump(ctx context.Context, dbconn Connection, opts DumpOpts, writers []DumpW
 				Routines:            opts.Routines,
 				SuppressUseDatabase: opts.SuppressUseDatabase,
 				MaxAllowedPacket:    opts.MaxAllowedPacket,
+				PostDumpDelay:       opts.PostDumpDelay,
 			}
 			if err := dumper.Dump(); err != nil {
 				return fmt.Errorf("failed to dump database %s: %v", schema, err)

pkg/database/mysql/dump.go

@@ -52,6 +52,7 @@ type Data struct {
 	SuppressUseDatabase bool
 	Charset             string
 	Collation           string
+	PostDumpDelay       time.Duration
 
 	tx                 *sql.Tx
 	headerTmpl         *template.Template
@@ -254,6 +255,9 @@ func (data *Data) Dump() error {
 		return data.err
 	}
 
+	if data.PostDumpDelay > 0 {
+		time.Sleep(data.PostDumpDelay)
+	}
 	meta.CompleteTime = time.Now().UTC().Format("2006-01-02 15:04:05")
 	return data.footerTmpl.Execute(data.Out, meta)
 }

pkg/database/restore.go

@@ -19,12 +19,11 @@ var (
 	createRegex = regexp.MustCompile(`(?i)^(CREATE\s+DATABASE\s*(\/\*.*\*\/\s*)?` + "`" + `)([^\s]+)(` + "`" + `\s*(\s*\/\*.*\*\/\s*)?\s*;$)`)
 )
 
-func Restore(ctx context.Context, dbconn Connection, databasesMap map[string]string, readers []io.ReadSeeker) error {
-	db, err := sql.Open("mysql", dbconn.MySQL())
+func Restore(ctx context.Context, dbconn *Connection, databasesMap map[string]string, readers []io.ReadSeeker) error {
+	db, err := dbconn.MySQL()
 	if err != nil {
 		return fmt.Errorf("failed to open connection to database: %v", err)
 	}
-	defer func() { _ = db.Close() }()
 
 	// load data into database by reading from each reader
 	for _, r := range readers {

pkg/database/schemas.go

@@ -1,7 +1,6 @@
 package database
 
 import (
-	"database/sql"
 	"fmt"
 )
 
@@ -16,12 +15,11 @@ func init() {
 	}
 }
 
-func GetSchemas(dbconn Connection) ([]string, error) {
-	db, err := sql.Open("mysql", dbconn.MySQL())
+func GetSchemas(dbconn *Connection) ([]string, error) {
+	db, err := dbconn.MySQL()
 	if err != nil {
 		return nil, fmt.Errorf("failed to open connection to database: %v", err)
 	}
-	defer func() { _ = db.Close() }()
 
 	// TODO: get list of schemas
 	// mysql -h $DB_SERVER -P $DB_PORT $DBUSER $DBPASS -N -e 'show databases'