Skip to content

Commit

Permalink
util: improve TopSQL reporter memory efficiency (#25195) (#25363)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-srebot
ti-srebot committed Jun 11, 2021
1 parent da126da commit 29bfbb2
Show file tree
Hide file tree
Showing 14 changed files with 586 additions and 231 deletions.
13 changes: 13 additions & 0 deletions domain/domain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -927,6 +927,19 @@ func (do *Domain) LoadSysVarCacheLoop(ctx sessionctx.Context) error {
case _, ok = <-watchCh:
case <-time.After(duration):
}

failpoint.Inject("skipLoadSysVarCacheLoop", func(val failpoint.Value) {
// In some pkg integration test, there are many testSuite, and each testSuite has separate storage and
// `LoadSysVarCacheLoop` background goroutine. Then each testSuite `RebuildSysVarCache` from it's
// own storage.
// Each testSuit will also call `checkEnableServerGlobalVar` to update some local variables.
// That's the problem, each testSuit use different storage to update some same local variables.
// So just skip `RebuildSysVarCache` in some integration testing.
if val.(bool) {
failpoint.Continue()
}
})

if !ok {
logutil.BgLogger().Error("LoadSysVarCacheLoop loop watch channel closed")
watchCh = do.etcdClient.Watch(context.Background(), sysVarCacheKey)
Expand Down
31 changes: 31 additions & 0 deletions domain/sysvar_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package domain
import (
"context"
"fmt"
"strconv"
"sync"

"github.com/pingcap/tidb/sessionctx"
Expand Down Expand Up @@ -166,6 +167,36 @@ func checkEnableServerGlobalVar(name, sVal string) {
err = stmtsummary.StmtSummaryByDigestMap.SetMaxSQLLength(sVal, false)
case variable.TiDBCapturePlanBaseline:
variable.CapturePlanBaseline.Set(sVal, false)
case variable.TiDBEnableTopSQL:
variable.TopSQLVariable.Enable.Store(variable.TiDBOptOn(sVal))
case variable.TiDBTopSQLPrecisionSeconds:
var val int64
val, err = strconv.ParseInt(sVal, 10, 64)
if err != nil {
break
}
variable.TopSQLVariable.PrecisionSeconds.Store(val)
case variable.TiDBTopSQLMaxStatementCount:
var val int64
val, err = strconv.ParseInt(sVal, 10, 64)
if err != nil {
break
}
variable.TopSQLVariable.MaxStatementCount.Store(val)
case variable.TiDBTopSQLMaxCollect:
var val int64
val, err = strconv.ParseInt(sVal, 10, 64)
if err != nil {
break
}
variable.TopSQLVariable.MaxCollect.Store(val)
case variable.TiDBTopSQLReportIntervalSeconds:
var val int64
val, err = strconv.ParseInt(sVal, 10, 64)
if err != nil {
break
}
variable.TopSQLVariable.ReportIntervalSeconds.Store(val)
}
if err != nil {
logutil.BgLogger().Error(fmt.Sprintf("load global variable %s error", name), zap.Error(err))
Expand Down
67 changes: 22 additions & 45 deletions executor/set_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1367,12 +1367,6 @@ func (s *testSuite5) TestSetClusterConfigJSONData(c *C) {

func (s *testSerialSuite) TestSetTopSQLVariables(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("set @@tidb_enable_top_sql='On';")
tk.MustQuery("select @@tidb_enable_top_sql;").Check(testkit.Rows("1"))
c.Assert(variable.TopSQLVariable.Enable.Load(), IsTrue)
tk.MustExec("set @@tidb_enable_top_sql='off';")
tk.MustQuery("select @@tidb_enable_top_sql;").Check(testkit.Rows("0"))
c.Assert(variable.TopSQLVariable.Enable.Load(), IsFalse)
tk.MustExec("set @@global.tidb_enable_top_sql='On';")
tk.MustQuery("select @@global.tidb_enable_top_sql;").Check(testkit.Rows("1"))
c.Assert(variable.TopSQLVariable.Enable.Load(), IsTrue)
Expand All @@ -1386,62 +1380,41 @@ func (s *testSerialSuite) TestSetTopSQLVariables(c *C) {
tk.MustExec("set @@tidb_top_sql_agent_address='';")
tk.MustQuery("select @@tidb_top_sql_agent_address;").Check(testkit.Rows(""))
c.Assert(variable.TopSQLVariable.AgentAddress.Load(), Equals, "")
tk.MustExec("set @@global.tidb_top_sql_agent_address='127.0.0.1:4001';")
tk.MustQuery("select @@global.tidb_top_sql_agent_address;").Check(testkit.Rows("127.0.0.1:4001"))
c.Assert(variable.TopSQLVariable.AgentAddress.Load(), Equals, "127.0.0.1:4001")
tk.MustExec("set @@global.tidb_top_sql_agent_address='';")
tk.MustQuery("select @@global.tidb_top_sql_agent_address;").Check(testkit.Rows(""))
c.Assert(variable.TopSQLVariable.AgentAddress.Load(), Equals, "")

tk.MustExec("set @@tidb_top_sql_precision_seconds=60;")
tk.MustQuery("select @@tidb_top_sql_precision_seconds;").Check(testkit.Rows("60"))
c.Assert(variable.TopSQLVariable.PrecisionSeconds.Load(), Equals, int64(60))
_, err := tk.Exec("set @@tidb_top_sql_precision_seconds='abc';")
c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_precision_seconds'")
_, err = tk.Exec("set @@tidb_top_sql_precision_seconds='-1';")
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_precision_seconds' can't be set to the value of '-1'")
tk.MustQuery("select @@tidb_top_sql_precision_seconds;").Check(testkit.Rows("60"))
c.Assert(variable.TopSQLVariable.PrecisionSeconds.Load(), Equals, int64(60))
tk.MustExec("set @@global.tidb_top_sql_precision_seconds=2;")
tk.MustQuery("select @@global.tidb_top_sql_precision_seconds;").Check(testkit.Rows("2"))
c.Assert(variable.TopSQLVariable.PrecisionSeconds.Load(), Equals, int64(2))
_, err = tk.Exec("set @@global.tidb_top_sql_precision_seconds='abc';")
_, err := tk.Exec("set @@global.tidb_top_sql_precision_seconds='abc';")
c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_precision_seconds'")
_, err = tk.Exec("set @@global.tidb_top_sql_precision_seconds='-1';")
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_precision_seconds' can't be set to the value of '-1'")
tk.MustQuery("select @@global.tidb_top_sql_precision_seconds;").Check(testkit.Rows("2"))
c.Assert(variable.TopSQLVariable.PrecisionSeconds.Load(), Equals, int64(2))

tk.MustExec("set @@tidb_top_sql_max_statement_count=5000;")
tk.MustQuery("select @@tidb_top_sql_max_statement_count;").Check(testkit.Rows("5000"))
c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(5000))
_, err = tk.Exec("set @@tidb_top_sql_max_statement_count='abc';")
c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_max_statement_count'")
_, err = tk.Exec("set @@tidb_top_sql_max_statement_count='-1';")
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_max_statement_count' can't be set to the value of '-1'")
tk.MustQuery("select @@tidb_top_sql_max_statement_count;").Check(testkit.Rows("5000"))
c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(5000))
tk.MustExec("set @@global.tidb_top_sql_max_statement_count=2;")
tk.MustQuery("select @@global.tidb_top_sql_max_statement_count;").Check(testkit.Rows("2"))
c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(2))
tk.MustExec("set @@global.tidb_top_sql_max_statement_count=20;")
tk.MustQuery("select @@global.tidb_top_sql_max_statement_count;").Check(testkit.Rows("20"))
c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(20))
_, err = tk.Exec("set @@global.tidb_top_sql_max_statement_count='abc';")
c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_max_statement_count'")
_, err = tk.Exec("set @@global.tidb_top_sql_max_statement_count='-1';")
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_max_statement_count' can't be set to the value of '-1'")
_, err = tk.Exec("set @@global.tidb_top_sql_max_statement_count='5001';")
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_max_statement_count' can't be set to the value of '5001'")
tk.MustQuery("select @@global.tidb_top_sql_precision_seconds;").Check(testkit.Rows("2"))
c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(2))
tk.MustQuery("select @@global.tidb_top_sql_max_statement_count;").Check(testkit.Rows("20"))
c.Assert(variable.TopSQLVariable.MaxStatementCount.Load(), Equals, int64(20))

tk.MustExec("set @@global.tidb_top_sql_max_collect=20000;")
tk.MustQuery("select @@global.tidb_top_sql_max_collect;").Check(testkit.Rows("20000"))
c.Assert(variable.TopSQLVariable.MaxCollect.Load(), Equals, int64(20000))
_, err = tk.Exec("set @@global.tidb_top_sql_max_collect='abc';")
c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_max_collect'")
_, err = tk.Exec("set @@global.tidb_top_sql_max_collect='-1';")
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_max_collect' can't be set to the value of '-1'")
_, err = tk.Exec("set @@global.tidb_top_sql_max_collect='500001';")
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_max_collect' can't be set to the value of '500001'")
tk.MustQuery("select @@global.tidb_top_sql_max_collect;").Check(testkit.Rows("20000"))
c.Assert(variable.TopSQLVariable.MaxCollect.Load(), Equals, int64(20000))

tk.MustExec("set @@tidb_top_sql_report_interval_seconds=10;")
tk.MustQuery("select @@tidb_top_sql_report_interval_seconds;").Check(testkit.Rows("10"))
c.Assert(variable.TopSQLVariable.ReportIntervalSeconds.Load(), Equals, int64(10))
_, err = tk.Exec("set @@tidb_top_sql_report_interval_seconds='abc';")
c.Assert(err.Error(), Equals, "[variable:1232]Incorrect argument type to variable 'tidb_top_sql_report_interval_seconds'")
_, err = tk.Exec("set @@tidb_top_sql_report_interval_seconds='5000';")
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_report_interval_seconds' can't be set to the value of '5000'")
tk.MustQuery("select @@tidb_top_sql_report_interval_seconds;").Check(testkit.Rows("10"))
c.Assert(variable.TopSQLVariable.ReportIntervalSeconds.Load(), Equals, int64(10))
tk.MustExec("set @@global.tidb_top_sql_report_interval_seconds=120;")
tk.MustQuery("select @@global.tidb_top_sql_report_interval_seconds;").Check(testkit.Rows("120"))
c.Assert(variable.TopSQLVariable.ReportIntervalSeconds.Load(), Equals, int64(120))
Expand All @@ -1451,4 +1424,8 @@ func (s *testSerialSuite) TestSetTopSQLVariables(c *C) {
c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_top_sql_report_interval_seconds' can't be set to the value of '5000'")
tk.MustQuery("select @@global.tidb_top_sql_report_interval_seconds;").Check(testkit.Rows("120"))
c.Assert(variable.TopSQLVariable.ReportIntervalSeconds.Load(), Equals, int64(120))

// Test for hide top sql variable in show variable.
tk.MustQuery("show variables like '%top_sql%'").Check(testkit.Rows())
tk.MustQuery("show global variables like '%top_sql%'").Check(testkit.Rows())
}
Loading

0 comments on commit 29bfbb2

Please sign in to comment.