From 6f3d54a13698b5079d285f8c2abadfd133021844 Mon Sep 17 00:00:00 2001 From: zj118228 Date: Tue, 23 Jan 2018 13:51:41 +0800 Subject: [PATCH 1/2] Aliyun RDS hide some mysql variables for secrity so gh-ost will get ilegal values. Add --aliyun-rds flag to avoid getting them. --- go/base/context.go | 3 +++ go/base/utils.go | 13 +++++++++++-- go/cmd/gh-ost/main.go | 2 ++ go/logic/applier.go | 6 ++++-- go/logic/inspect.go | 6 ++++-- go/mysql/utils.go | 6 ++++-- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 23fe6f6e2..5f63c88a9 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -91,6 +91,7 @@ type MigrationContext struct { SkipRenamedColumns bool IsTungsten bool DiscardForeignKeys bool + AliyunRDS bool config ContextConfig configMutex *sync.Mutex @@ -214,6 +215,8 @@ type ContextConfig struct { } } +var Context *MigrationContext + func NewMigrationContext() *MigrationContext { return &MigrationContext{ Uuid: uuid.NewV4().String(), diff --git a/go/base/utils.go b/go/base/utils.go index 727bc57e0..8d9e4f06a 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -65,16 +65,25 @@ func StringContainsAll(s string, substrings ...string) bool { } func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) (string, error) { - query := `select @@global.port, @@global.version` + versionQuery := `select @@global.version` var port, extraPort int var version string - if err := db.QueryRow(query).Scan(&port, &version); err != nil { + if err := db.QueryRow(versionQuery).Scan(&version); err != nil { return "", err } extraPortQuery := `select @@global.extra_port` if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil { // swallow this error. not all servers support extra_port } + // AliyunRDS set users port to "NULL", replace it by gh-ost param + if Context.AliyunRDS { + port = connectionConfig.Key.Port + } else { + portQuery := `select @@global.port` + if err := db.QueryRow(portQuery).Scan(&port); err != nil { + return "", err + } + } if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) { log.Infof("connection validated on %+v", connectionConfig.Key) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 74d0f0516..335d6cc4f 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -44,6 +44,7 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() + base.Context = migrationContext flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") @@ -68,6 +69,7 @@ func main() { flag.BoolVar(&migrationContext.IsTungsten, "tungsten", false, "explicitly let gh-ost know that you are running on a tungsten-replication based topology (you are likely to also provide --assume-master-host)") flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys") flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that") + flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/applier.go b/go/logic/applier.go index 227b59ec4..1a3e61863 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -89,10 +89,12 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.validateAndReadTimeZone(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { return err } else { - this.connectionConfig.ImpliedKey = impliedKey + if this.migrationContext.AliyunRDS != true { + this.connectionConfig.ImpliedKey = impliedKey + } } if err := this.readTableColumns(); err != nil { return err diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 31c81dce5..1417ffad9 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,10 +53,12 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { return err } else { - this.connectionConfig.ImpliedKey = impliedKey + if this.migrationContext.AliyunRDS != true { + this.connectionConfig.ImpliedKey = impliedKey + } } if err := this.validateGrants(); err != nil { return err diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4d5..c513ae596 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -171,9 +171,11 @@ func GetSelfBinlogCoordinates(db *gosql.DB) (selfBinlogCoordinates *BinlogCoordi } // GetInstanceKey reads hostname and port on given DB -func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { +func GetInstanceKey(db *gosql.DB, isAliyunRDS bool) (instanceKey *InstanceKey, err error) { instanceKey = &InstanceKey{} - err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) + if isAliyunRDS != true { + err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) + } return instanceKey, err } From d3b98d9e746115834e5b9dd64e29a157eddcd854 Mon Sep 17 00:00:00 2001 From: zj118228 Date: Thu, 29 Mar 2018 11:19:20 +0800 Subject: [PATCH 2/2] Refactor the code and remove the global variable of support-aliyun-rds branch. --- go/base/context.go | 2 -- go/base/utils.go | 4 ++-- go/cmd/gh-ost/main.go | 2 -- go/logic/applier.go | 12 ++++++------ go/logic/inspect.go | 10 +++++----- go/logic/streamer.go | 2 +- go/mysql/utils.go | 6 ++---- 7 files changed, 16 insertions(+), 22 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 5f63c88a9..981c386e8 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -215,8 +215,6 @@ type ContextConfig struct { } } -var Context *MigrationContext - func NewMigrationContext() *MigrationContext { return &MigrationContext{ Uuid: uuid.NewV4().String(), diff --git a/go/base/utils.go b/go/base/utils.go index 8d9e4f06a..d6bbf479c 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -64,7 +64,7 @@ func StringContainsAll(s string, substrings ...string) bool { return nonEmptyStringsFound } -func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) (string, error) { +func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext) (string, error) { versionQuery := `select @@global.version` var port, extraPort int var version string @@ -76,7 +76,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) // swallow this error. not all servers support extra_port } // AliyunRDS set users port to "NULL", replace it by gh-ost param - if Context.AliyunRDS { + if migrationContext.AliyunRDS { port = connectionConfig.Key.Port } else { portQuery := `select @@global.port` diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 335d6cc4f..70a4e9018 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -44,8 +44,6 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() - base.Context = migrationContext - flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") diff --git a/go/logic/applier.go b/go/logic/applier.go index 1a3e61863..0a2951ebe 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -78,21 +78,21 @@ func (this *Applier) InitDBConnections() (err error) { return err } this.singletonDB.SetMaxOpenConns(1) - version, err := base.ValidateConnection(this.db, this.connectionConfig) + version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext) if err != nil { return err } - if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig); err != nil { + if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig, this.migrationContext); err != nil { return err } this.migrationContext.ApplierMySQLVersion = version if err := this.validateAndReadTimeZone(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { - return err - } else { - if this.migrationContext.AliyunRDS != true { + if !this.migrationContext.AliyunRDS { + if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + return err + } else { this.connectionConfig.ImpliedKey = impliedKey } } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 1417ffad9..bb574479b 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,10 +53,10 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { - return err - } else { - if this.migrationContext.AliyunRDS != true { + if !this.migrationContext.AliyunRDS { + if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + return err + } else { this.connectionConfig.ImpliedKey = impliedKey } } @@ -205,7 +205,7 @@ func (this *Inspector) validateConnection() error { return fmt.Errorf("MySQL replication length limited to 32 characters. See https://dev.mysql.com/doc/refman/5.7/en/assigning-passwords.html") } - version, err := base.ValidateConnection(this.db, this.connectionConfig) + version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext) this.migrationContext.InspectorMySQLVersion = version return err } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 37e719543..25d9f536c 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -107,7 +107,7 @@ func (this *EventsStreamer) InitDBConnections() (err error) { if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, EventsStreamerUri); err != nil { return err } - if _, err := base.ValidateConnection(this.db, this.connectionConfig); err != nil { + if _, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext); err != nil { return err } if err := this.readCurrentBinlogCoordinates(); err != nil { diff --git a/go/mysql/utils.go b/go/mysql/utils.go index c513ae596..532cbb4d5 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -171,11 +171,9 @@ func GetSelfBinlogCoordinates(db *gosql.DB) (selfBinlogCoordinates *BinlogCoordi } // GetInstanceKey reads hostname and port on given DB -func GetInstanceKey(db *gosql.DB, isAliyunRDS bool) (instanceKey *InstanceKey, err error) { +func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { instanceKey = &InstanceKey{} - if isAliyunRDS != true { - err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) - } + err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) return instanceKey, err }