Skip to content

Commit

Permalink
This is an automated cherry-pick of pingcap#40182
Browse files Browse the repository at this point in the history 
Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>
  • Loading branch information
@chrysan @ti-chi-bot
chrysan authored and ti-chi-bot committed Feb 16, 2023
1 parent 8a40c4d commit c98fcd1
Show file tree
Hide file tree
Showing 3 changed files with 307 additions and 12 deletions.
2 changes: 1 addition & 1 deletion server/tidb_library_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func TestMemoryLeak(t *testing.T) {
runtime.ReadMemStats(&memStat)
// before the fix, initAndCloseTiDB for 20 times will cost 900 MB memory, so we test for a quite loose upper bound.
if syncutil.EnableDeadlock {
require.Less(t, memStat.HeapInuse-oldHeapInUse, uint64(1100*units.MiB))
require.Less(t, memStat.HeapInuse-oldHeapInUse, uint64(1500*units.MiB))
} else {
require.Less(t, memStat.HeapInuse-oldHeapInUse, uint64(300*units.MiB))
}
Expand Down
75 changes: 64 additions & 11 deletions session/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -786,19 +786,42 @@ const (
version107 = 107
// version108 adds the table tidb_ttl_table_status
version108 = 108
// version109 add column source to mysql.stats_meta_history
// version109 sets tidb_enable_gc_aware_memory_track to off when a cluster upgrades from some version lower than v6.5.0.
version109 = 109
<<<<<<< HEAD
// version110 sets tidb_enable_gc_aware_memory_track to off when a cluster upgrades from some version lower than v6.5.0.
version110 = 110
// version111 adds the table tidb_ttl_task and tidb_ttl_job_history
version111 = 111
// version112 modifies the view tidb_mdl_view
version112 = 112
=======
// ...
// [version110, version129] is the version range reserved for patches of 6.5.x
// ...
// version130 add column source to mysql.stats_meta_history
version130 = 130
// version131 adds the table tidb_ttl_task and tidb_ttl_job_history
version131 = 131
// version132 modifies the view tidb_mdl_view
version132 = 132
// version133 sets tidb_server_memory_limit to "80%"
version133 = 133
// version134 modifies the following global variables default value:
// - foreign_key_checks: off -> on
// - tidb_enable_foreign_key: off -> on
// - tidb_store_batch_size: 0 -> 4
version134 = 134
>>>>>>> 5df1913e0f6 (session: fix upgrade (#40182))
)

// currentBootstrapVersion is defined as a variable, so we can modify its value for testing.
// please make sure this is the largest version
<<<<<<< HEAD
var currentBootstrapVersion int64 = version112
=======
var currentBootstrapVersion int64 = version134
>>>>>>> 5df1913e0f6 (session: fix upgrade (#40182))

// DDL owner key's expired time is ManagerSessionTTL seconds, we should wait the time and give more time to have a chance to finish it.
var internalSQLTimeout = owner.ManagerSessionTTL + 15
Expand Down Expand Up @@ -915,9 +938,17 @@ var (
upgradeToVer107,
upgradeToVer108,
upgradeToVer109,
<<<<<<< HEAD
upgradeToVer110,
upgradeToVer111,
upgradeToVer112,
=======
upgradeToVer130,
upgradeToVer131,
upgradeToVer132,
upgradeToVer133,
upgradeToVer134,
>>>>>>> 5df1913e0f6 (session: fix upgrade (#40182))
}
)

Expand Down Expand Up @@ -2254,37 +2285,59 @@ func upgradeToVer108(s Session, ver int64) {
doReentrantDDL(s, CreateTTLTableStatus)
}

// For users that upgrade TiDB from a 6.2-6.4 version, we want to disable tidb gc_aware_memory_track by default.
func upgradeToVer109(s Session, ver int64) {
if ver >= version109 {
return
}
doReentrantDDL(s, "ALTER TABLE mysql.stats_meta_history ADD COLUMN IF NOT EXISTS `source` varchar(40) NOT NULL after `version`;")
mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);",
mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBEnableGCAwareMemoryTrack, 0)
}

// For users that upgrade TiDB from a 6.2-6.4 version, we want to disable tidb gc_aware_memory_track by default.
func upgradeToVer110(s Session, ver int64) {
if ver >= version110 {
func upgradeToVer130(s Session, ver int64) {
if ver >= version130 {
return
}
mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);",
mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBEnableGCAwareMemoryTrack, 0)
doReentrantDDL(s, "ALTER TABLE mysql.stats_meta_history ADD COLUMN IF NOT EXISTS `source` varchar(40) NOT NULL after `version`;")
}

func upgradeToVer111(s Session, ver int64) {
if ver >= version111 {
func upgradeToVer131(s Session, ver int64) {
if ver >= version131 {
return
}
doReentrantDDL(s, CreateTTLTask)
doReentrantDDL(s, CreateTTLJobHistory)
}

func upgradeToVer112(s Session, ver int64) {
if ver >= version112 {
func upgradeToVer132(s Session, ver int64) {
if ver >= version132 {
return
}
doReentrantDDL(s, CreateMDLView)
}

<<<<<<< HEAD
=======
func upgradeToVer133(s Session, ver int64) {
if ver >= version133 {
return
}
mustExecute(s, "UPDATE HIGH_PRIORITY %n.%n set VARIABLE_VALUE = %? where VARIABLE_NAME = %? and VARIABLE_VALUE = %?;",
mysql.SystemDB, mysql.GlobalVariablesTable, variable.DefTiDBServerMemoryLimit, variable.TiDBServerMemoryLimit, "0")
}

func upgradeToVer134(s Session, ver int64) {
if ver >= version134 {
return
}
mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, variable.ForeignKeyChecks, variable.On)
mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBEnableForeignKey, variable.On)
mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBEnableHistoricalStats, variable.On)
mustExecute(s, "REPLACE HIGH_PRIORITY INTO %n.%n VALUES (%?, %?);", mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBEnablePlanReplayerCapture, variable.On)
mustExecute(s, "UPDATE HIGH_PRIORITY %n.%n SET VARIABLE_VALUE = %? WHERE VARIABLE_NAME = %? AND VARIABLE_VALUE = %?;", mysql.SystemDB, mysql.GlobalVariablesTable, "4", variable.TiDBStoreBatchSize, "0")
}

>>>>>>> 5df1913e0f6 (session: fix upgrade (#40182))
func writeOOMAction(s Session) {
comment := "oom-action is `log` by default in v3.0.x, `cancel` by default in v4.0.11+"
mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`,
Expand Down
242 changes: 242 additions & 0 deletions session/bootstrap_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1417,3 +1417,245 @@ func TestTiDBGCAwareUpgradeFrom630To650(t *testing.T) {
require.Equal(t, 2, row.Len())
require.Equal(t, "0", row.GetString(1))
}
<<<<<<< HEAD
=======

func TestTiDBServerMemoryLimitUpgradeTo651_1(t *testing.T) {
ctx := context.Background()
store, _ := createStoreAndBootstrap(t)
defer func() { require.NoError(t, store.Close()) }()

// upgrade from 6.5.0 to 6.5.1+.
ver132 := version132
seV132 := createSessionAndSetID(t, store)
txn, err := store.Begin()
require.NoError(t, err)
m := meta.NewMeta(txn)
err = m.FinishBootstrap(int64(ver132))
require.NoError(t, err)
err = txn.Commit(context.Background())
require.NoError(t, err)
mustExec(t, seV132, fmt.Sprintf("update mysql.tidb set variable_value=%d where variable_name='tidb_server_version'", ver132))
mustExec(t, seV132, fmt.Sprintf("update mysql.GLOBAL_VARIABLES set variable_value='%s' where variable_name='%s'", "0", variable.TiDBServerMemoryLimit))
mustExec(t, seV132, "commit")
unsetStoreBootstrapped(store.UUID())
ver, err := getBootstrapVersion(seV132)
require.NoError(t, err)
require.Equal(t, int64(ver132), ver)

// We are now in 6.5.0, tidb_server_memory_limit is 0.
res := mustExecToRecodeSet(t, seV132, fmt.Sprintf("select * from mysql.GLOBAL_VARIABLES where variable_name='%s'", variable.TiDBServerMemoryLimit))
chk := res.NewChunk(nil)
err = res.Next(ctx, chk)
require.NoError(t, err)
require.Equal(t, 1, chk.NumRows())
row := chk.GetRow(0)
require.Equal(t, 2, row.Len())
require.Equal(t, "0", row.GetString(1))

// Upgrade to 6.5.1+.
domCurVer, err := BootstrapSession(store)
require.NoError(t, err)
defer domCurVer.Close()
seCurVer := createSessionAndSetID(t, store)
ver, err = getBootstrapVersion(seCurVer)
require.NoError(t, err)
require.Equal(t, currentBootstrapVersion, ver)

// We are now in 6.5.1+.
res = mustExecToRecodeSet(t, seCurVer, fmt.Sprintf("select * from mysql.GLOBAL_VARIABLES where variable_name='%s'", variable.TiDBServerMemoryLimit))
chk = res.NewChunk(nil)
err = res.Next(ctx, chk)
require.NoError(t, err)
require.Equal(t, 1, chk.NumRows())
row = chk.GetRow(0)
require.Equal(t, 2, row.Len())
require.Equal(t, variable.DefTiDBServerMemoryLimit, row.GetString(1))
}

func TestTiDBServerMemoryLimitUpgradeTo651_2(t *testing.T) {
ctx := context.Background()
store, _ := createStoreAndBootstrap(t)
defer func() { require.NoError(t, store.Close()) }()

// upgrade from 6.5.0 to 6.5.1+.
ver132 := version132
seV132 := createSessionAndSetID(t, store)
txn, err := store.Begin()
require.NoError(t, err)
m := meta.NewMeta(txn)
err = m.FinishBootstrap(int64(ver132))
require.NoError(t, err)
err = txn.Commit(context.Background())
require.NoError(t, err)
mustExec(t, seV132, fmt.Sprintf("update mysql.tidb set variable_value=%d where variable_name='tidb_server_version'", ver132))
mustExec(t, seV132, fmt.Sprintf("update mysql.GLOBAL_VARIABLES set variable_value='%s' where variable_name='%s'", "70%", variable.TiDBServerMemoryLimit))
mustExec(t, seV132, "commit")
unsetStoreBootstrapped(store.UUID())
ver, err := getBootstrapVersion(seV132)
require.NoError(t, err)
require.Equal(t, int64(ver132), ver)

// We are now in 6.5.0, tidb_server_memory_limit is "70%".
res := mustExecToRecodeSet(t, seV132, fmt.Sprintf("select * from mysql.GLOBAL_VARIABLES where variable_name='%s'", variable.TiDBServerMemoryLimit))
chk := res.NewChunk(nil)
err = res.Next(ctx, chk)
require.NoError(t, err)
require.Equal(t, 1, chk.NumRows())
row := chk.GetRow(0)
require.Equal(t, 2, row.Len())
require.Equal(t, "70%", row.GetString(1))

// Upgrade to 6.5.1+.
domCurVer, err := BootstrapSession(store)
require.NoError(t, err)
defer domCurVer.Close()
seCurVer := createSessionAndSetID(t, store)
ver, err = getBootstrapVersion(seCurVer)
require.NoError(t, err)
require.Equal(t, currentBootstrapVersion, ver)

// We are now in 6.5.1+.
res = mustExecToRecodeSet(t, seCurVer, fmt.Sprintf("select * from mysql.GLOBAL_VARIABLES where variable_name='%s'", variable.TiDBServerMemoryLimit))
chk = res.NewChunk(nil)
err = res.Next(ctx, chk)
require.NoError(t, err)
require.Equal(t, 1, chk.NumRows())
row = chk.GetRow(0)
require.Equal(t, 2, row.Len())
require.Equal(t, "70%", row.GetString(1))
}

func TestTiDBGlobalVariablesDefaultValueUpgradeFrom630To660(t *testing.T) {
ctx := context.Background()
store, _ := createStoreAndBootstrap(t)
defer func() { require.NoError(t, store.Close()) }()

// upgrade from 6.3.0 to 6.6.0.
ver630 := version93
seV630 := createSessionAndSetID(t, store)
txn, err := store.Begin()
require.NoError(t, err)
m := meta.NewMeta(txn)
err = m.FinishBootstrap(int64(ver630))
require.NoError(t, err)
err = txn.Commit(context.Background())
require.NoError(t, err)
mustExec(t, seV630, fmt.Sprintf("update mysql.tidb set variable_value=%d where variable_name='tidb_server_version'", ver630))
mustExec(t, seV630, fmt.Sprintf("update mysql.GLOBAL_VARIABLES set variable_value='%s' where variable_name='%s'", "OFF", variable.TiDBEnableForeignKey))
mustExec(t, seV630, fmt.Sprintf("update mysql.GLOBAL_VARIABLES set variable_value='%s' where variable_name='%s'", "OFF", variable.ForeignKeyChecks))
mustExec(t, seV630, fmt.Sprintf("update mysql.GLOBAL_VARIABLES set variable_value='%s' where variable_name='%s'", "OFF", variable.TiDBEnableHistoricalStats))
mustExec(t, seV630, fmt.Sprintf("update mysql.GLOBAL_VARIABLES set variable_value='%s' where variable_name='%s'", "OFF", variable.TiDBEnablePlanReplayerCapture))
mustExec(t, seV630, "commit")
unsetStoreBootstrapped(store.UUID())
ver, err := getBootstrapVersion(seV630)
require.NoError(t, err)
require.Equal(t, int64(ver630), ver)

// We are now in 6.3.0.
upgradeVars := []string{variable.TiDBEnableForeignKey, variable.ForeignKeyChecks, variable.TiDBEnableHistoricalStats, variable.TiDBEnablePlanReplayerCapture}
varsValueList := []string{"OFF", "OFF", "OFF", "OFF"}
for i := range upgradeVars {
res := mustExecToRecodeSet(t, seV630, fmt.Sprintf("select * from mysql.GLOBAL_VARIABLES where variable_name='%s'", upgradeVars[i]))
chk := res.NewChunk(nil)
err = res.Next(ctx, chk)
require.NoError(t, err)
require.Equal(t, 1, chk.NumRows())
row := chk.GetRow(0)
require.Equal(t, 2, row.Len())
require.Equal(t, varsValueList[i], row.GetString(1))
}

// Upgrade to 6.6.0.
domCurVer, err := BootstrapSession(store)
require.NoError(t, err)
defer domCurVer.Close()
seV660 := createSessionAndSetID(t, store)
ver, err = getBootstrapVersion(seV660)
require.NoError(t, err)
require.Equal(t, currentBootstrapVersion, ver)

// We are now in 6.6.0.
varsValueList = []string{"ON", "ON", "ON", "ON"}
for i := range upgradeVars {
res := mustExecToRecodeSet(t, seV660, fmt.Sprintf("select * from mysql.GLOBAL_VARIABLES where variable_name='%s'", upgradeVars[i]))
chk := res.NewChunk(nil)
err = res.Next(ctx, chk)
require.NoError(t, err)
require.Equal(t, 1, chk.NumRows())
row := chk.GetRow(0)
require.Equal(t, 2, row.Len())
require.Equal(t, varsValueList[i], row.GetString(1))
}
}

func TestTiDBStoreBatchSizeUpgradeFrom650To660(t *testing.T) {
for i := 0; i < 2; i++ {
func() {
ctx := context.Background()
store, dom := createStoreAndBootstrap(t)
defer func() { require.NoError(t, store.Close()) }()

// upgrade from 6.5 to 6.6.
ver65 := version132
seV65 := createSessionAndSetID(t, store)
txn, err := store.Begin()
require.NoError(t, err)
m := meta.NewMeta(txn)
err = m.FinishBootstrap(int64(ver65))
require.NoError(t, err)
err = txn.Commit(context.Background())
require.NoError(t, err)
mustExec(t, seV65, fmt.Sprintf("update mysql.tidb set variable_value=%d where variable_name='tidb_server_version'", ver65))
mustExec(t, seV65, fmt.Sprintf("update mysql.GLOBAL_VARIABLES set variable_value='%s' where variable_name='%s'", "0", variable.TiDBStoreBatchSize))
mustExec(t, seV65, "commit")
unsetStoreBootstrapped(store.UUID())
ver, err := getBootstrapVersion(seV65)
require.NoError(t, err)
require.Equal(t, int64(ver65), ver)

// We are now in 6.5, tidb_store_batch_size is 0.
res := mustExecToRecodeSet(t, seV65, fmt.Sprintf("select * from mysql.GLOBAL_VARIABLES where variable_name='%s'", variable.TiDBStoreBatchSize))
chk := res.NewChunk(nil)
err = res.Next(ctx, chk)
require.NoError(t, err)
require.Equal(t, 1, chk.NumRows())
row := chk.GetRow(0)
require.Equal(t, 2, row.Len())
require.Equal(t, "0", row.GetString(1))
res.Close()

if i == 0 {
// For the first time, We set tidb_store_batch_size to 1.
// And after upgrade to 6.6, tidb_store_batch_size should be 1.
// For the second it should be the latest default value.
mustExec(t, seV65, "set global tidb_store_batch_size = 1")
}
dom.Close()
// Upgrade to 6.6.
domCurVer, err := BootstrapSession(store)
require.NoError(t, err)
defer domCurVer.Close()
seCurVer := createSessionAndSetID(t, store)
ver, err = getBootstrapVersion(seCurVer)
require.NoError(t, err)
require.Equal(t, currentBootstrapVersion, ver)

// We are now in 6.6.
res = mustExecToRecodeSet(t, seCurVer, "select @@tidb_store_batch_size")
chk = res.NewChunk(nil)
err = res.Next(ctx, chk)
require.NoError(t, err)
require.Equal(t, 1, chk.NumRows())
row = chk.GetRow(0)
require.Equal(t, 1, row.Len())
if i == 0 {
require.Equal(t, "1", row.GetString(0))
} else {
require.Equal(t, "4", row.GetString(0))
}
res.Close()
}()
}
}
>>>>>>> 5df1913e0f6 (session: fix upgrade (#40182))

0 comments on commit c98fcd1

Please sign in to comment.