Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: github/gh-ost
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: master
Choose a base ref
...
head repository: bytebase/gh-ost2
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: main
Choose a head ref
  • 8 commits
  • 16 files changed
  • 3 contributors

Commits on Feb 19, 2025

  1. feat: change the prefix to

    @d-bytebase @RainbowDashy
    d-bytebase authored and RainbowDashy committed Feb 19, 2025
    Copy the full SHA
    a05e231 View commit details

  2. chore: no need to set tx isolation level

    @RainbowDashy
    RainbowDashy committed Feb 19, 2025
    Copy the full SHA
    513890e View commit details

  3. chore: teardown and return on panicAbort

    @RainbowDashy
    RainbowDashy committed Feb 19, 2025
    Copy the full SHA
    33cb385 View commit details

  4. chore: use uuid as tls config key (#4)

    @RainbowDashy
    RainbowDashy committed Feb 19, 2025
    Copy the full SHA
    bc2fd04 View commit details

  5. feat: ghost database name

    @RainbowDashy
    RainbowDashy committed Feb 19, 2025
    Copy the full SHA
    07c908d View commit details

Commits on Feb 21, 2025

  1. fix: nil unixListener

    @RainbowDashy
    RainbowDashy committed Feb 21, 2025
    Copy the full SHA
    d3c3cc3 View commit details

Commits on Mar 10, 2025

  1. fix: close binlog reader on teardown

    @RainbowDashy
    RainbowDashy committed Mar 10, 2025
    Copy the full SHA
    240bd56 View commit details

  2. Merge pull request #5 from bytebase/binl 

    fix: close binlog reader on teardown
    @d-bytebase
    d-bytebase authored Mar 10, 2025

    Verified

    This commit was created on GitHub.com and signed with GitHub’s verified signature.
    GPG key ID: B5690EEEBB952194
    Verified
    Learn about vigilant mode
    Copy the full SHA
    33e16ca View commit details
7 changes: 4 additions & 3 deletions go/base/context.go
View file
Original file line number Diff line number Diff line change
@@ -77,6 +77,7 @@ func NewThrottleCheckResult(throttle bool, reason string, reasonHint ThrottleRea
type MigrationContext struct {
Uuid string

GhostDatabaseName string
DatabaseName string
OriginalTableName string
AlterStatement string
@@ -322,12 +323,12 @@ func (this *MigrationContext) SetConnectionCharset(charset string) {
}

func getSafeTableName(baseName string, suffix string) string {
name := fmt.Sprintf("_%s_%s", baseName, suffix)
name := fmt.Sprintf("~%s_%s", baseName, suffix)
if len(name) <= mysql.MaxTableNameLength {
return name
}
extraCharacters := len(name) - mysql.MaxTableNameLength
return fmt.Sprintf("_%s_%s", baseName[0:len(baseName)-extraCharacters], suffix)
return fmt.Sprintf("~%s_%s", baseName[0:len(baseName)-extraCharacters], suffix)
}

// GetGhostTableName generates the name of ghost table, based on original table name
@@ -865,7 +866,7 @@ func (this *MigrationContext) ApplyCredentials() {

func (this *MigrationContext) SetupTLS() error {
if this.UseTLS {
return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate, this.TLSCertificate, this.TLSKey, this.TLSAllowInsecure)
return this.InspectorConnectionConfig.UseTLS(this.Uuid, this.TLSCACertificate, this.TLSCertificate, this.TLSKey, this.TLSAllowInsecure)
}
return nil
}
22 changes: 11 additions & 11 deletions go/base/context_test.go
View file
Original file line number Diff line number Diff line change
@@ -22,22 +22,22 @@ func TestGetTableNames(t *testing.T) {
{
context := NewMigrationContext()
context.OriginalTableName = "some_table"
require.Equal(t, "_some_table_del", context.GetOldTableName())
require.Equal(t, "_some_table_gho", context.GetGhostTableName())
require.Equal(t, "_some_table_ghc", context.GetChangelogTableName(), "_some_table_ghc")
require.Equal(t, "~some_table_del", context.GetOldTableName())
require.Equal(t, "~some_table_gho", context.GetGhostTableName())
require.Equal(t, "~some_table_ghc", context.GetChangelogTableName(), "~some_table_ghc")
}
{
context := NewMigrationContext()
context.OriginalTableName = "a123456789012345678901234567890123456789012345678901234567890"
require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_del", context.GetOldTableName())
require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_gho", context.GetGhostTableName())
require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_ghc", context.GetChangelogTableName())
require.Equal(t, "~a1234567890123456789012345678901234567890123456789012345678_del", context.GetOldTableName())
require.Equal(t, "~a1234567890123456789012345678901234567890123456789012345678_gho", context.GetGhostTableName())
require.Equal(t, "~a1234567890123456789012345678901234567890123456789012345678_ghc", context.GetChangelogTableName())
}
{
context := NewMigrationContext()
context.OriginalTableName = "a123456789012345678901234567890123456789012345678901234567890123"
oldTableName := context.GetOldTableName()
require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_del", oldTableName)
require.Equal(t, "~a1234567890123456789012345678901234567890123456789012345678_del", oldTableName)
}
{
context := NewMigrationContext()
@@ -46,15 +46,15 @@ func TestGetTableNames(t *testing.T) {
longForm := "Jan 2, 2006 at 3:04pm (MST)"
context.StartTime, _ = time.Parse(longForm, "Feb 3, 2013 at 7:54pm (PST)")
oldTableName := context.GetOldTableName()
require.Equal(t, "_a1234567890123456789012345678901234567890123_20130203195400_del", oldTableName)
require.Equal(t, "~a1234567890123456789012345678901234567890123_20130203195400_del", oldTableName)
}
{
context := NewMigrationContext()
context.OriginalTableName = "foo_bar_baz"
context.ForceTmpTableName = "tmp"
require.Equal(t, "_tmp_del", context.GetOldTableName())
require.Equal(t, "_tmp_gho", context.GetGhostTableName())
require.Equal(t, "_tmp_ghc", context.GetChangelogTableName())
require.Equal(t, "~tmp_del", context.GetOldTableName())
require.Equal(t, "~tmp_gho", context.GetGhostTableName())
require.Equal(t, "~tmp_ghc", context.GetChangelogTableName())
}
}

79 changes: 41 additions & 38 deletions go/logic/applier.go
View file
Original file line number Diff line number Diff line change
@@ -116,15 +116,15 @@ func (this *Applier) InitDBConnections() (err error) {

func (this *Applier) prepareQueries() (err error) {
if this.dmlDeleteQueryBuilder, err = sql.NewDMLDeleteQueryBuilder(
this.migrationContext.DatabaseName,
this.migrationContext.GhostDatabaseName,
this.migrationContext.GetGhostTableName(),
this.migrationContext.OriginalTableColumns,
&this.migrationContext.UniqueKey.Columns,
); err != nil {
return err
}
if this.dmlInsertQueryBuilder, err = sql.NewDMLInsertQueryBuilder(
this.migrationContext.DatabaseName,
this.migrationContext.GhostDatabaseName,
this.migrationContext.GetGhostTableName(),
this.migrationContext.OriginalTableColumns,
this.migrationContext.SharedColumns,
@@ -133,7 +133,7 @@ func (this *Applier) prepareQueries() (err error) {
return err
}
if this.dmlUpdateQueryBuilder, err = sql.NewDMLUpdateQueryBuilder(
this.migrationContext.DatabaseName,
this.migrationContext.GhostDatabaseName,
this.migrationContext.GetGhostTableName(),
this.migrationContext.OriginalTableColumns,
this.migrationContext.SharedColumns,
@@ -197,8 +197,8 @@ func (this *Applier) readTableColumns() (err error) {
}

// showTableStatus returns the output of `show table status like '...'` command
func (this *Applier) showTableStatus(tableName string) (rowMap sqlutils.RowMap) {
query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(this.migrationContext.DatabaseName), tableName)
func (this *Applier) showTableStatus(databaseName, tableName string) (rowMap sqlutils.RowMap) {
query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(databaseName), tableName)
sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error {
rowMap = m
return nil
@@ -207,8 +207,8 @@ func (this *Applier) showTableStatus(tableName string) (rowMap sqlutils.RowMap)
}

// tableExists checks if a given table exists in database
func (this *Applier) tableExists(tableName string) (tableFound bool) {
m := this.showTableStatus(tableName)
func (this *Applier) tableExists(databaseName, tableName string) (tableFound bool) {
m := this.showTableStatus(databaseName, tableName)
return (m != nil)
}

@@ -220,8 +220,8 @@ func (this *Applier) ValidateOrDropExistingTables() error {
return err
}
}
if this.tableExists(this.migrationContext.GetGhostTableName()) {
return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GetGhostTableName()))
if this.tableExists(this.migrationContext.GhostDatabaseName, this.migrationContext.GetGhostTableName()) {
return fmt.Errorf("Table %s.%s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GhostDatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()))
}
if this.migrationContext.InitiallyDropOldTable {
if err := this.DropOldTable(); err != nil {
@@ -232,8 +232,8 @@ func (this *Applier) ValidateOrDropExistingTables() error {
this.migrationContext.Log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength)
}

if this.tableExists(this.migrationContext.GetOldTableName()) {
return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-old-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GetOldTableName()))
if this.tableExists(this.migrationContext.GhostDatabaseName, this.migrationContext.GetOldTableName()) {
return fmt.Errorf("Table %s.%s already exists. Panicking. Use --initially-drop-old-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GhostDatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()))
}

return nil
@@ -272,13 +272,13 @@ func (this *Applier) AttemptInstantDDL() error {
// CreateGhostTable creates the ghost table on the applier host
func (this *Applier) CreateGhostTable() error {
query := fmt.Sprintf(`create /* gh-ost */ table %s.%s like %s.%s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
)
this.migrationContext.Log.Infof("Creating ghost table %s.%s",
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
)

@@ -313,12 +313,12 @@ func (this *Applier) CreateGhostTable() error {
// AlterGhost applies `alter` statement on ghost table
func (this *Applier) AlterGhost() error {
query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s %s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
this.migrationContext.AlterStatementOptions,
)
this.migrationContext.Log.Infof("Altering ghost table %s.%s",
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
)
this.migrationContext.Log.Debugf("ALTER statement: %s", query)
@@ -354,12 +354,12 @@ func (this *Applier) AlterGhost() error {
// AlterGhost applies `alter` statement on ghost table
func (this *Applier) AlterGhostAutoIncrement() error {
query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s AUTO_INCREMENT=%d`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
this.migrationContext.OriginalTableAutoIncrement,
)
this.migrationContext.Log.Infof("Altering ghost table AUTO_INCREMENT value %s.%s",
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
)
this.migrationContext.Log.Debugf("AUTO_INCREMENT ALTER statement: %s", query)
@@ -383,12 +383,12 @@ func (this *Applier) CreateChangelogTable() error {
primary key(id),
unique key hint_uidx(hint)
) auto_increment=256 comment='%s'`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetChangelogTableName()),
GhostChangelogTableComment,
)
this.migrationContext.Log.Infof("Creating changelog table %s.%s",
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetChangelogTableName()),
)
if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
@@ -401,11 +401,11 @@ func (this *Applier) CreateChangelogTable() error {
// dropTable drops a given table on the applied host
func (this *Applier) dropTable(tableName string) error {
query := fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(tableName),
)
this.migrationContext.Log.Infof("Dropping table %s.%s",
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(tableName),
)
if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
@@ -452,7 +452,7 @@ func (this *Applier) WriteChangelog(hint, value string) (string, error) {
on duplicate key update
last_update=NOW(),
value=VALUES(value)`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetChangelogTableName()),
)
_, err := sqlutils.ExecNoPrepare(this.db, query, explicitId, hint, value)
@@ -670,6 +670,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected
query, explodedArgs, err := sql.BuildRangeInsertPreparedQuery(
this.migrationContext.DatabaseName,
this.migrationContext.OriginalTableName,
this.migrationContext.GhostDatabaseName,
this.migrationContext.GetGhostTableName(),
this.migrationContext.SharedColumns.Names(),
this.migrationContext.MappedSharedColumns.Names(),
@@ -757,19 +758,21 @@ func (this *Applier) UnlockTables() error {
// - rename ghost table to original
// There is a point in time in between where the table does not exist.
func (this *Applier) SwapTablesQuickAndBumpy() error {
query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s rename %s`,
query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s rename %s.%s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetOldTableName()),
)
this.migrationContext.Log.Infof("Renaming original table")
this.migrationContext.RenameTablesStartTime = time.Now()
if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil {
return err
}
query = fmt.Sprintf(`alter /* gh-ost */ table %s.%s rename %s`,
sql.EscapeName(this.migrationContext.DatabaseName),
query = fmt.Sprintf(`alter /* gh-ost */ table %s.%s rename%s. %s`,
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
)
this.migrationContext.Log.Infof("Renaming ghost table")
@@ -790,9 +793,9 @@ func (this *Applier) RenameTablesRollback() (renameError error) {
query := fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s, %s.%s to %s.%s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetOldTableName()),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
@@ -805,15 +808,15 @@ func (this *Applier) RenameTablesRollback() (renameError error) {
query = fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
)
this.migrationContext.Log.Infof("Renaming back to ghost table")
if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
renameError = err
}
query = fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetOldTableName()),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
@@ -951,7 +954,7 @@ func (this *Applier) ExpectProcess(sessionId int64, stateHint, infoHint string)
func (this *Applier) DropAtomicCutOverSentryTableIfExists() error {
this.migrationContext.Log.Infof("Looking for magic cut-over table")
tableName := this.migrationContext.GetOldTableName()
rowMap := this.showTableStatus(tableName)
rowMap := this.showTableStatus(this.migrationContext.GhostDatabaseName, tableName)
if rowMap == nil {
// Table does not exist
return nil
@@ -974,13 +977,13 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error {
create /* gh-ost */ table %s.%s (
id int auto_increment primary key
) engine=%s comment='%s'`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(tableName),
this.migrationContext.TableEngine,
atomicCutOverMagicHint,
)
this.migrationContext.Log.Infof("Creating magic cut-over table %s.%s",
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(tableName),
)
if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
@@ -1066,13 +1069,13 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
query = fmt.Sprintf(`lock /* gh-ost */ tables %s.%s write, %s.%s write`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetOldTableName()),
)
this.migrationContext.Log.Infof("Locking %s.%s, %s.%s",
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetOldTableName()),
)
this.migrationContext.LockTablesStartTime = time.Now()
@@ -1095,7 +1098,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
// And in fact, we will:
this.migrationContext.Log.Infof("Dropping magic cut-over table")
query = fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetOldTableName()),
)

@@ -1108,7 +1111,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s",
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetOldTableName()),
)
query = `unlock /* gh-ost */ tables`
@@ -1147,9 +1150,9 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed
query = fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s, %s.%s to %s.%s`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetOldTableName()),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.GhostDatabaseName),
sql.EscapeName(this.migrationContext.GetGhostTableName()),
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
Loading