From 84cd49ade2e429ee92615c6f34aea054dbd7df5e Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 20 Jan 2023 01:44:26 +0800 Subject: [PATCH 01/24] *: let processlist support resource group and enhance drop resource group check Signed-off-by: BornChanger --- ddl/ddl_api.go | 14 +++++ ddl/resourcegroup/group.go | 4 +- infoschema/tables.go | 2 + infoschema/tables_test.go | 125 +++++++++++++++++++------------------ session/session.go | 2 + util/processinfo.go | 9 +-- 6 files changed, 90 insertions(+), 66 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 14b1f3247141e..492fa7bd38b55 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -7692,6 +7692,20 @@ func (d *ddl) DropResourceGroup(ctx sessionctx.Context, stmt *ast.DropResourceGr return err } + // check in cluster to see if any active thread is using the group, we have to use internal SQL here + exec := ctx.(sqlexec.RestrictedSQLExecutor) + internalCtx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) + sql := "select id, instance, user FROM information_schema.cluster_processlist where ResourceGroupName = \"" + groupName.L + "\" limit 1" + rows, _, err := exec.ExecRestrictedSQL(internalCtx, nil, sql) + if err != nil { + err = errors.Errorf("system table information_schema.cluster_processlist access failed") + return err + } + if len(rows) != 0 { + err = errors.Errorf("thread [%d] of user [%s] from instance [%s] is using the resource group to drop. Retry after the thread is terminated", rows[0].GetInt64(0), rows[0].GetString(1), rows[0].GetString(2)) + return err + } + job := &model.Job{ SchemaID: group.ID, SchemaName: group.Name.L, diff --git a/ddl/resourcegroup/group.go b/ddl/resourcegroup/group.go index 0023787cc777c..33fceefac6299 100644 --- a/ddl/resourcegroup/group.go +++ b/ddl/resourcegroup/group.go @@ -21,14 +21,14 @@ import ( "k8s.io/apimachinery/pkg/api/resource" ) -const maxGroupNameLength = 32 +const MaxGroupNameLength = 32 // NewGroupFromOptions creates a new resource group from the given options. func NewGroupFromOptions(groupName string, options *model.ResourceGroupSettings) (*rmpb.ResourceGroup, error) { if options == nil { return nil, ErrInvalidGroupSettings } - if len(groupName) > maxGroupNameLength { + if len(groupName) > MaxGroupNameLength { return nil, ErrTooLongResourceGroupName } group := &rmpb.ResourceGroup{ diff --git a/infoschema/tables.go b/infoschema/tables.go index a38319a05c63b..365d29f60ea96 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/pingcap/tidb/ddl/resourcegroup" "net" "net/http" "strconv" @@ -824,6 +825,7 @@ var tableProcesslistCols = []columnInfo{ {name: "MEM", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "DISK", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "TxnStart", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, deflt: ""}, + {name: "ResourceGroupName", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag, deflt: ""}, } var tableTiDBIndexesCols = []columnInfo{ diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index f796345bbd8e7..a3c2494068fb9 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -303,87 +303,92 @@ func TestSomeTables(t *testing.T) { tk.SetSession(se) sm := &testkit.MockSessionManager{PS: make([]*util.ProcessInfo, 0)} sm.PS = append(sm.PS, &util.ProcessInfo{ - ID: 1, - User: "user-1", - Host: "localhost", - Port: "", - DB: "information_schema", - Command: byte(1), - Digest: "abc1", - State: 1, - Info: "do something", - StmtCtx: tk.Session().GetSessionVars().StmtCtx, + ID: 1, + User: "user-1", + Host: "localhost", + Port: "", + DB: "information_schema", + Command: byte(1), + Digest: "abc1", + State: 1, + Info: "do something", + StmtCtx: tk.Session().GetSessionVars().StmtCtx, + ResourceGroupName: "rg1", }) sm.PS = append(sm.PS, &util.ProcessInfo{ - ID: 2, - User: "user-2", - Host: "localhost", - Port: "", - DB: "test", - Command: byte(2), - Digest: "abc2", - State: 2, - Info: strings.Repeat("x", 101), - StmtCtx: tk.Session().GetSessionVars().StmtCtx, + ID: 2, + User: "user-2", + Host: "localhost", + Port: "", + DB: "test", + Command: byte(2), + Digest: "abc2", + State: 2, + Info: strings.Repeat("x", 101), + StmtCtx: tk.Session().GetSessionVars().StmtCtx, + ResourceGroupName: "rg2", }) sm.PS = append(sm.PS, &util.ProcessInfo{ - ID: 3, - User: "user-3", - Host: "127.0.0.1", - Port: "12345", - DB: "test", - Command: byte(2), - Digest: "abc3", - State: 1, - Info: "check port", - StmtCtx: tk.Session().GetSessionVars().StmtCtx, + ID: 3, + User: "user-3", + Host: "127.0.0.1", + Port: "12345", + DB: "test", + Command: byte(2), + Digest: "abc3", + State: 1, + Info: "check port", + StmtCtx: tk.Session().GetSessionVars().StmtCtx, + ResourceGroupName: "rg3", }) tk.Session().SetSessionManager(sm) tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 ", "in transaction", "do something"), - fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s abc2 0 0 ", "autocommit", strings.Repeat("x", 101)), - fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s abc3 0 0 ", "in transaction", "check port"), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1 ", "in transaction", "do something"), + fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s abc2 0 0 rg2 ", "autocommit", strings.Repeat("x", 101)), + fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s abc3 0 0 rg3 ", "in transaction", "check port"), )) tk.MustQuery("SHOW PROCESSLIST;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s", "in transaction", "do something"), - fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s", "autocommit", strings.Repeat("x", 100)), - fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s", "in transaction", "check port"), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s rg1", "in transaction", "do something"), + fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s rg2", "autocommit", strings.Repeat("x", 100)), + fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s rg3", "in transaction", "check port"), )) tk.MustQuery("SHOW FULL PROCESSLIST;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s", "in transaction", "do something"), - fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s", "autocommit", strings.Repeat("x", 101)), - fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s", "in transaction", "check port"), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s rg1", "in transaction", "do something"), + fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s rg2", "autocommit", strings.Repeat("x", 101)), + fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s rg3", "in transaction", "check port"), )) sm = &testkit.MockSessionManager{PS: make([]*util.ProcessInfo, 0)} sm.PS = append(sm.PS, &util.ProcessInfo{ - ID: 1, - User: "user-1", - Host: "localhost", - DB: "information_schema", - Command: byte(1), - Digest: "abc1", - State: 1, + ID: 1, + User: "user-1", + Host: "localhost", + DB: "information_schema", + Command: byte(1), + Digest: "abc1", + State: 1, + ResourceGroupName: "rg1", }) sm.PS = append(sm.PS, &util.ProcessInfo{ - ID: 2, - User: "user-2", - Host: "localhost", - Command: byte(2), - Digest: "abc2", - State: 2, - Info: strings.Repeat("x", 101), - CurTxnStartTS: 410090409861578752, + ID: 2, + User: "user-2", + Host: "localhost", + Command: byte(2), + Digest: "abc2", + State: 2, + Info: strings.Repeat("x", 101), + CurTxnStartTS: 410090409861578752, + ResourceGroupName: "rg2", }) tk.Session().SetSessionManager(sm) tk.Session().GetSessionVars().TimeZone = time.UTC tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 ", "in transaction", ""), - fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752)", "autocommit", strings.Repeat("x", 101)), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1 ", "in transaction", ""), + fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752) rg2", "autocommit", strings.Repeat("x", 101)), )) tk.MustQuery("SHOW PROCESSLIST;").Sort().Check( testkit.Rows( @@ -392,16 +397,16 @@ func TestSomeTables(t *testing.T) { )) tk.MustQuery("SHOW FULL PROCESSLIST;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s", "in transaction", ""), - fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s", "autocommit", strings.Repeat("x", 101)), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s rg1", "in transaction", ""), + fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s rg2", "autocommit", strings.Repeat("x", 101)), )) tk.MustQuery("select * from information_schema.PROCESSLIST where db is null;").Check( testkit.Rows( - fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752)", "autocommit", strings.Repeat("x", 101)), + fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752) rg2", "autocommit", strings.Repeat("x", 101)), )) tk.MustQuery("select * from information_schema.PROCESSLIST where Info is null;").Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 ", "in transaction", ""), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1 ", "in transaction", ""), )) } diff --git a/session/session.go b/session/session.go index 5b7fa5a8875aa..c9b0ccbd9833b 100644 --- a/session/session.go +++ b/session/session.go @@ -1574,6 +1574,7 @@ func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecu if explain, ok := p.(*plannercore.Explain); ok && explain.Analyze && explain.TargetPlan != nil { p = explain.TargetPlan } + pi := util.ProcessInfo{ ID: s.sessionVars.ConnectionID, Port: s.sessionVars.Port, @@ -1595,6 +1596,7 @@ func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecu MaxExecutionTime: maxExecutionTime, RedactSQL: s.sessionVars.EnableRedactLog, ProtectedTSList: &s.sessionVars.ProtectedTSList, + ResourceGroupName: s.sessionVars.ResourceGroupName, } oldPi := s.ShowProcess() if p == nil { diff --git a/util/processinfo.go b/util/processinfo.go index dee4f4ea30a53..47e960601c75a 100644 --- a/util/processinfo.go +++ b/util/processinfo.go @@ -70,10 +70,11 @@ type ProcessInfo struct { CurTxnStartTS uint64 // MaxExecutionTime is the timeout for select statement, in milliseconds. // If the query takes too long, kill it. - MaxExecutionTime uint64 - State uint16 - Command byte - RedactSQL bool + MaxExecutionTime uint64 + State uint16 + Command byte + RedactSQL bool + ResourceGroupName string } // ToRowForShow returns []interface{} for the row data of "SHOW [FULL] PROCESSLIST". From 122bec6c20ecda90072a91a1cd69d40fccf9995c Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 20 Jan 2023 02:02:38 +0800 Subject: [PATCH 02/24] *: make lint happy Signed-off-by: BornChanger --- ddl/resourcegroup/group.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ddl/resourcegroup/group.go b/ddl/resourcegroup/group.go index 33fceefac6299..17c4a0afd407a 100644 --- a/ddl/resourcegroup/group.go +++ b/ddl/resourcegroup/group.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" ) +// MaxGroupNameLength is max length of the name of a resource group const MaxGroupNameLength = 32 // NewGroupFromOptions creates a new resource group from the given options. From 5b9388e0cf33f21284f6e08d3d20be0df0cd0b8e Mon Sep 17 00:00:00 2001 From: BornChanger Date: Sat, 28 Jan 2023 15:58:10 +0800 Subject: [PATCH 03/24] *: polish code Signed-off-by: BornChanger --- executor/simple.go | 11 ++++---- .../simpletest/password_management_test.go | 2 +- infoschema/tables.go | 2 +- infoschema/tables_test.go | 26 +++++++++---------- parser/ast/misc.go | 7 ++--- parser/parser.go | 2 +- parser/parser.y | 2 +- privilege/privileges/privileges_test.go | 2 +- util/processinfo.go | 2 +- 9 files changed, 26 insertions(+), 30 deletions(-) diff --git a/executor/simple.go b/executor/simple.go index 2beab893bb5f9..94394571b9599 100644 --- a/executor/simple.go +++ b/executor/simple.go @@ -1090,14 +1090,13 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm userAttributes = append(userAttributes, fmt.Sprintf("\"metadata\": %s", s.CommentOrAttributeOption.Value)) } } - resourceGroupName := "default" + if s.ResourceGroupNameOption != nil { if !variable.EnableResourceControl.Load() { return infoschema.ErrResourceGroupSupportDisabled } - if s.ResourceGroupNameOption.Type == ast.UserResourceGroupName { - resourceGroupName = s.ResourceGroupNameOption.Value - } + + resourceGroupName := s.ResourceGroupNameOption.Value // check if specified resource group exists if resourceGroupName != "default" && resourceGroupName != "" { @@ -1106,8 +1105,8 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm return infoschema.ErrResourceGroupNotExists } } + userAttributes = append(userAttributes, fmt.Sprintf("\"resource_group\": \"%s\"", resourceGroupName)) } - userAttributes = append(userAttributes, fmt.Sprintf("\"resource_group\": \"%s\"", resourceGroupName)) // If FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME are both specified to 0, a string of 0 length is generated. // When inserting the attempts into json, an error occurs. This requires special handling. if PasswordLocking != "" { @@ -1904,7 +1903,7 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt) newAttributes = append(newAttributes, fmt.Sprintf(`"metadata": %s`, s.CommentOrAttributeOption.Value)) } } - if s.ResourceGroupNameOption != nil && s.ResourceGroupNameOption.Type == ast.UserResourceGroupName { + if s.ResourceGroupNameOption != nil { if !variable.EnableResourceControl.Load() { return infoschema.ErrResourceGroupSupportDisabled } diff --git a/executor/simpletest/password_management_test.go b/executor/simpletest/password_management_test.go index ae3352cdf84b7..c3368162ce877 100644 --- a/executor/simpletest/password_management_test.go +++ b/executor/simpletest/password_management_test.go @@ -1227,7 +1227,7 @@ func TestPasswordExpiredAndTacking(t *testing.T) { require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password("qwe123"), nil)) tk = testkit.NewTestKit(t, store) createAndCheck(tk, `CREATE USER 'u3'@'localhost' IDENTIFIED BY '!@#HASHhs123' PASSWORD EXPIRE INTERVAL 3 DAY FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME 3 COMMENT 'Some statements to test create user'`, - "{\"Password_locking\": {\"failed_login_attempts\": 4, \"password_lock_time_days\": 3}, \"metadata\": {\"comment\": \"Some statements to test create user\"}, \"resource_group\": \"default\"}", user) + "{\"Password_locking\": {\"failed_login_attempts\": 4, \"password_lock_time_days\": 3}, \"metadata\": {\"comment\": \"Some statements to test create user\"}}", user) require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password("!@#HASHhs123"), nil)) tk = testkit.NewTestKit(t, store) diff --git a/infoschema/tables.go b/infoschema/tables.go index 365d29f60ea96..1c8aed2500291 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -825,7 +825,7 @@ var tableProcesslistCols = []columnInfo{ {name: "MEM", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "DISK", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "TxnStart", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, deflt: ""}, - {name: "ResourceGroupName", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag, deflt: ""}, + {name: "RESOURCE_GROUP_NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag, deflt: ""}, } var tableTiDBIndexesCols = []columnInfo{ diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index a3c2494068fb9..f08ad0c82a2b4 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -344,21 +344,21 @@ func TestSomeTables(t *testing.T) { tk.Session().SetSessionManager(sm) tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1 ", "in transaction", "do something"), - fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s abc2 0 0 rg2 ", "autocommit", strings.Repeat("x", 101)), - fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s abc3 0 0 rg3 ", "in transaction", "check port"), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1", "in transaction", "do something"), + fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s abc2 0 0 rg2", "autocommit", strings.Repeat("x", 101)), + fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s abc3 0 0 rg3", "in transaction", "check port"), )) tk.MustQuery("SHOW PROCESSLIST;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s rg1", "in transaction", "do something"), - fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s rg2", "autocommit", strings.Repeat("x", 100)), - fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s rg3", "in transaction", "check port"), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s", "in transaction", "do something"), + fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s", "autocommit", strings.Repeat("x", 100)), + fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s", "in transaction", "check port"), )) tk.MustQuery("SHOW FULL PROCESSLIST;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s rg1", "in transaction", "do something"), - fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s rg2", "autocommit", strings.Repeat("x", 101)), - fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s rg3", "in transaction", "check port"), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s", "in transaction", "do something"), + fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 %s %s", "autocommit", strings.Repeat("x", 101)), + fmt.Sprintf("3 user-3 127.0.0.1:12345 test Init DB 9223372036 %s %s", "in transaction", "check port"), )) sm = &testkit.MockSessionManager{PS: make([]*util.ProcessInfo, 0)} @@ -387,7 +387,7 @@ func TestSomeTables(t *testing.T) { tk.Session().GetSessionVars().TimeZone = time.UTC tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1 ", "in transaction", ""), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1", "in transaction", ""), fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s abc2 0 0 07-29 03:26:05.158(410090409861578752) rg2", "autocommit", strings.Repeat("x", 101)), )) tk.MustQuery("SHOW PROCESSLIST;").Sort().Check( @@ -397,8 +397,8 @@ func TestSomeTables(t *testing.T) { )) tk.MustQuery("SHOW FULL PROCESSLIST;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s rg1", "in transaction", ""), - fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s rg2", "autocommit", strings.Repeat("x", 101)), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s", "in transaction", ""), + fmt.Sprintf("2 user-2 localhost Init DB 9223372036 %s %s", "autocommit", strings.Repeat("x", 101)), )) tk.MustQuery("select * from information_schema.PROCESSLIST where db is null;").Check( testkit.Rows( @@ -406,7 +406,7 @@ func TestSomeTables(t *testing.T) { )) tk.MustQuery("select * from information_schema.PROCESSLIST where Info is null;").Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1 ", "in transaction", ""), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 %s %s abc1 0 0 rg1", "in transaction", ""), )) } diff --git a/parser/ast/misc.go b/parser/ast/misc.go index 8d5ecd3fdb4de..eec57ae2a172c 100644 --- a/parser/ast/misc.go +++ b/parser/ast/misc.go @@ -1584,15 +1584,12 @@ func (c *CommentOrAttributeOption) Restore(ctx *format.RestoreCtx) error { } type ResourceGroupNameOption struct { - Type int Value string } func (c *ResourceGroupNameOption) Restore(ctx *format.RestoreCtx) error { - if c.Type == UserResourceGroupName { - ctx.WriteKeyWord(" RESOURCE GROUP ") - ctx.WriteName(c.Value) - } + ctx.WriteKeyWord(" RESOURCE GROUP ") + ctx.WriteName(c.Value) return nil } diff --git a/parser/parser.go b/parser/parser.go index a463737e298da..6ac03e1139c6f 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -21292,7 +21292,7 @@ yynewstate: } case 2381: { - parser.yyVAL.item = &ast.ResourceGroupNameOption{Type: ast.UserResourceGroupName, Value: yyS[yypt-0].ident} + parser.yyVAL.item = &ast.ResourceGroupNameOption{Value: yyS[yypt-0].ident} } case 2382: { diff --git a/parser/parser.y b/parser/parser.y index 6a2974a80122d..abce90d9e3c71 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -13041,7 +13041,7 @@ ResourceGroupNameOption: } | "RESOURCE" "GROUP" ResourceGroupName { - $$ = &ast.ResourceGroupNameOption{Type: ast.UserResourceGroupName, Value: $3} + $$ = &ast.ResourceGroupNameOption{Value: $3} } PasswordOrLockOptions: diff --git a/privilege/privileges/privileges_test.go b/privilege/privileges/privileges_test.go index 1bbfe0e852022..0040751e0ff9f 100644 --- a/privilege/privileges/privileges_test.go +++ b/privilege/privileges/privileges_test.go @@ -1137,7 +1137,7 @@ func TestCreateDropUser(t *testing.T) { tk.MustExec(`DROP USER tcd3`) tk.MustExec(`CREATE USER usr1`) - tk.MustQuery(`SELECT User_attributes FROM mysql.user WHERE User = "usr1"`).Check(testkit.Rows("{\"resource_group\": \"default\"}")) + tk.MustQuery(`SELECT User_attributes FROM mysql.user WHERE User = "usr1"`).Check(testkit.Rows("{}")) tk.MustExec(`DROP USER usr1`) tk.MustExec("set global tidb_enable_resource_control = 'on'") diff --git a/util/processinfo.go b/util/processinfo.go index 47e960601c75a..37548cc586762 100644 --- a/util/processinfo.go +++ b/util/processinfo.go @@ -136,7 +136,7 @@ func (pi *ProcessInfo) ToRow(tz *time.Location) []interface{} { diskConsumed = pi.DiskTracker.BytesConsumed() } } - return append(pi.ToRowForShow(true), pi.Digest, bytesConsumed, diskConsumed, pi.txnStartTs(tz)) + return append(pi.ToRowForShow(true), pi.Digest, bytesConsumed, diskConsumed, pi.txnStartTs(tz), pi.ResourceGroupName) } // GetMinStartTS returns the minimum start-ts (used to delay GC) that greater than `lowerBound` (0 if no such one). From fb8f294194ec88c03a6497a35afd1c7dae95567f Mon Sep 17 00:00:00 2001 From: BornChanger Date: Sat, 28 Jan 2023 16:43:37 +0800 Subject: [PATCH 04/24] *: make bazel happy Signed-off-by: BornChanger --- ddl/ddl_api.go | 3 +-- infoschema/BUILD.bazel | 1 + infoschema/tables.go | 2 +- util/processinfo.go | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 492fa7bd38b55..92f727e7e0066 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -7695,8 +7695,7 @@ func (d *ddl) DropResourceGroup(ctx sessionctx.Context, stmt *ast.DropResourceGr // check in cluster to see if any active thread is using the group, we have to use internal SQL here exec := ctx.(sqlexec.RestrictedSQLExecutor) internalCtx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) - sql := "select id, instance, user FROM information_schema.cluster_processlist where ResourceGroupName = \"" + groupName.L + "\" limit 1" - rows, _, err := exec.ExecRestrictedSQL(internalCtx, nil, sql) + rows, _, err := exec.ExecRestrictedSQL(internalCtx, nil, "select id, instance, user FROM information_schema.cluster_processlist where ResourceGroupName = \"%?\" limit 1", groupName.L) if err != nil { err = errors.Errorf("system table information_schema.cluster_processlist access failed") return err diff --git a/infoschema/BUILD.bazel b/infoschema/BUILD.bazel index f827394be7904..20a4f150e4a27 100644 --- a/infoschema/BUILD.bazel +++ b/infoschema/BUILD.bazel @@ -17,6 +17,7 @@ go_library( deps = [ "//config", "//ddl/placement", + "//ddl/resourcegroup", "//domain/infosync", "//errno", "//kv", diff --git a/infoschema/tables.go b/infoschema/tables.go index 1c8aed2500291..e262da273cf3a 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/pingcap/tidb/ddl/resourcegroup" "net" "net/http" "strconv" @@ -33,6 +32,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/placement" + "github.com/pingcap/tidb/ddl/resourcegroup" "github.com/pingcap/tidb/domain/infosync" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" diff --git a/util/processinfo.go b/util/processinfo.go index 37548cc586762..135b1c58d5676 100644 --- a/util/processinfo.go +++ b/util/processinfo.go @@ -64,17 +64,17 @@ type ProcessInfo struct { User string Info string Port string + ResourceGroupName string PlanExplainRows [][]string OOMAlarmVariablesInfo OOMAlarmVariablesInfo ID uint64 CurTxnStartTS uint64 // MaxExecutionTime is the timeout for select statement, in milliseconds. // If the query takes too long, kill it. - MaxExecutionTime uint64 - State uint16 - Command byte - RedactSQL bool - ResourceGroupName string + MaxExecutionTime uint64 + State uint16 + Command byte + RedactSQL bool } // ToRowForShow returns []interface{} for the row data of "SHOW [FULL] PROCESSLIST". From 7c0b4a2ad7b569103e88ff2bad6d029f248b50ad Mon Sep 17 00:00:00 2001 From: BornChanger Date: Sat, 28 Jan 2023 19:00:10 +0800 Subject: [PATCH 05/24] *: fix cases Signed-off-by: BornChanger --- ddl/ddl_api.go | 13 ----- ddl/resource_group_test.go | 3 ++ executor/simple_test.go | 8 ++-- .../simpletest/password_management_test.go | 48 +++++++++---------- infoschema/tables_test.go | 3 +- util/gpool/gpool.go | 2 +- util/gpool/spmc/spmcpool.go | 21 ++++---- 7 files changed, 44 insertions(+), 54 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 92f727e7e0066..14b1f3247141e 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -7692,19 +7692,6 @@ func (d *ddl) DropResourceGroup(ctx sessionctx.Context, stmt *ast.DropResourceGr return err } - // check in cluster to see if any active thread is using the group, we have to use internal SQL here - exec := ctx.(sqlexec.RestrictedSQLExecutor) - internalCtx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) - rows, _, err := exec.ExecRestrictedSQL(internalCtx, nil, "select id, instance, user FROM information_schema.cluster_processlist where ResourceGroupName = \"%?\" limit 1", groupName.L) - if err != nil { - err = errors.Errorf("system table information_schema.cluster_processlist access failed") - return err - } - if len(rows) != 0 { - err = errors.Errorf("thread [%d] of user [%s] from instance [%s] is using the resource group to drop. Retry after the thread is terminated", rows[0].GetInt64(0), rows[0].GetString(1), rows[0].GetString(2)) - return err - } - job := &model.Job{ SchemaID: group.ID, SchemaName: group.Name.L, diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index 2a396488483d8..3607f9503da96 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -159,7 +159,10 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustExec("create resource group do_not_delete_rg ru_per_sec=100") tk.MustExec("create user usr3 resource group do_not_delete_rg") + tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "do_not_delete_rg"}`)) tk.MustContainErrMsg("drop resource group do_not_delete_rg", "user [usr3] depends on the resource group to drop") + tk.MustExec("alter user usr3 resource group `default`") + tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "default"}`)) } func testResourceGroupNameFromIS(t *testing.T, ctx sessionctx.Context, name string) *model.ResourceGroupInfo { diff --git a/executor/simple_test.go b/executor/simple_test.go index 31f2b719d84bc..0ededa68e2253 100644 --- a/executor/simple_test.go +++ b/executor/simple_test.go @@ -94,9 +94,9 @@ func TestUserAttributes(t *testing.T) { _, err := rootTK.Exec(`CREATE USER testuser2 ATTRIBUTE '{"name": "Tom", age: 19}'`) rootTK.MustExec(`CREATE USER testuser2`) require.Error(t, err) - rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser'`).Check(testkit.Rows(`{"metadata": {"comment": "1234"}, "resource_group": "default"}`)) - rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser1'`).Check(testkit.Rows(`{"metadata": {"age": 19, "name": "Tom"}, "resource_group": "default"}`)) - rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser2'`).Check(testkit.Rows(`{"resource_group": "default"}`)) + rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser'`).Check(testkit.Rows(`{"metadata": {"comment": "1234"}}`)) + rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser1'`).Check(testkit.Rows(`{"metadata": {"age": 19, "name": "Tom"}}`)) + rootTK.MustQuery(`SELECT user_attributes FROM mysql.user WHERE user = 'testuser2'`).Check(testkit.Rows(`{}`)) rootTK.MustQueryWithContext(ctx, `SELECT attribute FROM information_schema.user_attributes WHERE user = 'testuser'`).Check(testkit.Rows(`{"comment": "1234"}`)) rootTK.MustQueryWithContext(ctx, `SELECT attribute FROM information_schema.user_attributes WHERE user = 'testuser1'`).Check(testkit.Rows(`{"age": 19, "name": "Tom"}`)) rootTK.MustQueryWithContext(ctx, `SELECT attribute->>"$.age" AS age, attribute->>"$.name" AS name FROM information_schema.user_attributes WHERE user = 'testuser1'`).Check(testkit.Rows(`19 Tom`)) @@ -127,7 +127,7 @@ func TestUserAttributes(t *testing.T) { // https://github.com/pingcap/tidb/issues/39207 rootTK.MustExec("create user usr1@'%' identified by 'passord'") rootTK.MustExec("alter user usr1 comment 'comment1'") - rootTK.MustQuery("select user_attributes from mysql.user where user = 'usr1'").Check(testkit.Rows(`{"metadata": {"comment": "comment1"}, "resource_group": "default"}`)) + rootTK.MustQuery("select user_attributes from mysql.user where user = 'usr1'").Check(testkit.Rows(`{"metadata": {"comment": "comment1"}}`)) rootTK.MustExec("set global tidb_enable_resource_control = 'on'") rootTK.MustExec("CREATE RESOURCE GROUP rg1 ru_per_sec = 100") rootTK.MustExec("alter user usr1 resource group rg1") diff --git a/executor/simpletest/password_management_test.go b/executor/simpletest/password_management_test.go index c3368162ce877..1e79662b5d5a3 100644 --- a/executor/simpletest/password_management_test.go +++ b/executor/simpletest/password_management_test.go @@ -407,11 +407,11 @@ func TestFailedLoginTrackingBasic(t *testing.T) { testkit.Rows("CREATE USER 'u8'@'localhost' IDENTIFIED WITH 'mysql_native_password' AS '*2470C0C06DEE42FD1618BB99005ADCA2EC9D1E19' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK PASSWORD HISTORY DEFAULT PASSWORD REUSE INTERVAL DEFAULT FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME UNBOUNDED")) tk.MustExec("ALTER USER 'u4'@'localhost' PASSWORD_LOCK_TIME 0 FAILED_LOGIN_ATTEMPTS 0") - tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(`{"resource_group": "default"}`)) + tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(``)) tk.MustExec("ALTER USER 'u4'@'localhost' account unlock") - tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(`{"resource_group": "default"}`)) + tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(``)) tk.MustExec("ALTER USER 'u4'@'localhost' PASSWORD_LOCK_TIME 6") - tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(`{"Password_locking": {"failed_login_attempts": 0, "password_lock_time_days": 6}, "resource_group": "default"}`)) + tk.MustQuery("select user_attributes from mysql.user where user = 'u4' and host = 'localhost'").Check(testkit.Rows(`{"Password_locking": {"failed_login_attempts": 0, "password_lock_time_days": 6}}`)) } func TestUserReuseControl(t *testing.T) { @@ -748,7 +748,7 @@ func TestFailedLoginTracking(t *testing.T) { // Set FAILED_LOGIN_ATTEMPTS to 1, and check error messages after login failure once. createAndCheck(tk, "CREATE USER 'testu1'@'localhost' IDENTIFIED BY 'testu1' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1", - "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 1}, \"resource_group\": \"default\"}", "testu1") + "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 1}}", "testu1") err := tk.Session().Auth(&auth.UserIdentity{Username: "testu1", Hostname: "localhost"}, sha1Password("password"), nil) lds := strconv.FormatInt(1, 10) errTarget := privileges.GenerateAccountAutoLockErr(1, "testu1", "localhost", lds, lds) @@ -762,7 +762,7 @@ func TestFailedLoginTracking(t *testing.T) { // Set FAILED_LOGIN_ATTEMPTS to 1 and PASSWORD_LOCK_TIME to UNBOUNDED. Check error messages after failed login once. createAndCheck(tk, "CREATE USER 'testu2'@'localhost' IDENTIFIED BY 'testu2' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME UNBOUNDED", - "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": -1}, \"resource_group\": \"default\"}", "testu2") + "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": -1}}", "testu2") err = tk.Session().Auth(&auth.UserIdentity{Username: "testu2", Hostname: "localhost"}, sha1Password("password"), nil) errTarget = privileges.GenerateAccountAutoLockErr(1, "testu2", "localhost", "unlimited", "unlimited") require.Equal(t, err.Error(), errTarget.Error()) @@ -775,31 +775,31 @@ func TestFailedLoginTracking(t *testing.T) { // Set FAILED_LOGIN_ATTEMPTS to 0 or PASSWORD_LOCK_TIME to 0. Check error messages after failed login once. createAndCheck(tk, "CREATE USER 'testu3'@'localhost' IDENTIFIED BY 'testu3' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME UNBOUNDED", - "{\"Password_locking\": {\"failed_login_attempts\": 0, \"password_lock_time_days\": -1}, \"resource_group\": \"default\"}", "testu3") + "{\"Password_locking\": {\"failed_login_attempts\": 0, \"password_lock_time_days\": -1}}", "testu3") err = tk.Session().Auth(&auth.UserIdentity{Username: "testu3", Hostname: "localhost"}, sha1Password("password"), nil) require.ErrorContains(t, err, "Access denied for user 'testu3'@'localhost' (using password: YES)") checkAuthUser(t, tk, "testu3", 0, "") createAndCheck(tk, "CREATE USER 'testu4'@'localhost' IDENTIFIED BY 'testu4' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 0", - "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 0}, \"resource_group\": \"default\"}", "testu4") + "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 0}}", "testu4") err = tk.Session().Auth(&auth.UserIdentity{Username: "testu4", Hostname: "localhost"}, sha1Password("password"), nil) require.ErrorContains(t, err, "Access denied for user 'testu4'@'localhost' (using password: YES)") checkAuthUser(t, tk, "testu4", 0, "") tk.MustExec("CREATE USER 'testu5'@'localhost' IDENTIFIED BY 'testu5' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME 0") err = tk.Session().Auth(&auth.UserIdentity{Username: "testu5", Hostname: "localhost"}, sha1Password("password"), nil) require.ErrorContains(t, err, "Access denied for user 'testu5'@'localhost' (using password: YES)") - tk.MustQuery("select user_attributes from mysql.user where user= 'testu5' and host = 'localhost'").Check(testkit.Rows("{\"resource_group\": \"default\"}")) + tk.MustQuery("select user_attributes from mysql.user where user= 'testu5' and host = 'localhost'").Check(testkit.Rows("{}")) tk.MustExec("DROP USER 'testu1'@'localhost', 'testu2'@'localhost', 'testu3'@'localhost', 'testu4'@'localhost', 'testu5'@'localhost'") // Create user specifying only comment. tk.MustExec("CREATE USER 'testu1'@'localhost' IDENTIFIED BY 'testu1' comment 'testcomment' ") tk.MustQuery("select user_attributes from mysql.user where user= 'testu1' and host = 'localhost'"). - Check(testkit.Rows("{\"metadata\": {\"comment\": \"testcomment\"}, \"resource_group\": \"default\"}")) + Check(testkit.Rows("{\"metadata\": {\"comment\": \"testcomment\"}}")) // Create user specifying only attribute. tk.MustExec("create user testu2@'localhost' identified by 'testu2' ATTRIBUTE '{\"attribute\":\"testattribute\"}'") tk.MustQuery("select user_attributes from mysql.user where user= 'testu2' and host = 'localhost'"). - Check(testkit.Rows("{\"metadata\": {\"attribute\": \"testattribute\"}, \"resource_group\": \"default\"}")) + Check(testkit.Rows("{\"metadata\": {\"attribute\": \"testattribute\"}}")) // Create user specified comment and FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME. tk.MustExec("create user testu3@'localhost' identified by 'testu3' FAILED_LOGIN_ATTEMPTS 1 " + @@ -932,9 +932,9 @@ func TestFailedLoginTracking(t *testing.T) { rootk := testkit.NewTestKit(t, store) createAndCheck(tk, "CREATE USER 'u6'@'localhost' IDENTIFIED BY '' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 3", - "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 3}, \"resource_group\": \"default\"}", "u6") + "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 3}}", "u6") createAndCheck(tk, "CREATE USER 'u5'@'localhost' IDENTIFIED BY '' FAILED_LOGIN_ATTEMPTS 60 PASSWORD_LOCK_TIME 3", - "{\"Password_locking\": {\"failed_login_attempts\": 60, \"password_lock_time_days\": 3}, \"resource_group\": \"default\"}", "u5") + "{\"Password_locking\": {\"failed_login_attempts\": 60, \"password_lock_time_days\": 3}}", "u5") require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil)) checkAuthUser(t, rootk, "u6", 1, "N") @@ -984,7 +984,7 @@ func TestFailedLoginTracking(t *testing.T) { checkAuthUser(t, rootk, "u6", 1, "N") createAndCheck(rootk, "CREATE USER 'u1'@'localhost' IDENTIFIED BY '' FAILED_LOGIN_ATTEMPTS 3", - "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 0}, \"resource_group\": \"default\"}", "u1") + "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 0}}", "u1") require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: "u6", Hostname: "localhost"}, sha1Password("password"), nil)) checkAuthUser(t, rootk, "u1", 0, "") alterAndCheck(t, rootk, "ALTER USER 'u1'@'localhost' PASSWORD_LOCK_TIME 6;", "u1", 3, 6, 0) @@ -999,18 +999,18 @@ func TestFailedLoginTrackingAlterUser(t *testing.T) { // Create user specifying only comment. tk.MustExec("CREATE USER 'testu1'@'localhost' IDENTIFIED BY 'testu1' comment 'testcomment' ") tk.MustQuery("select user_attributes from mysql.user where user= 'testu1' and host = 'localhost'"). - Check(testkit.Rows("{\"metadata\": {\"comment\": \"testcomment\"}, \"resource_group\": \"default\"}")) + Check(testkit.Rows("{\"metadata\": {\"comment\": \"testcomment\"}}")) tk.MustExec("Alter USER 'testu1'@'localhost' comment ''") tk.MustQuery("select user_attributes from mysql.user where user= 'testu1' and host = 'localhost'"). - Check(testkit.Rows("{\"metadata\": {\"comment\": \"\"}, \"resource_group\": \"default\"}")) + Check(testkit.Rows("{\"metadata\": {\"comment\": \"\"}}")) // Create user specifying only attribute. tk.MustExec("CREATE USER 'testu2'@'localhost' IDENTIFIED BY 'testu2' ATTRIBUTE '{\"attribute\":\"testattribute\"}'") tk.MustQuery("select user_attributes from mysql.user where user= 'testu2' and host = 'localhost'"). - Check(testkit.Rows("{\"metadata\": {\"attribute\": \"testattribute\"}, \"resource_group\": \"default\"}")) + Check(testkit.Rows("{\"metadata\": {\"attribute\": \"testattribute\"}}")) tk.MustExec("Alter USER 'testu2'@'localhost' ATTRIBUTE '{\"attribute\":\"test\"}'") tk.MustQuery("select user_attributes from mysql.user where user= 'testu2' and host = 'localhost'"). - Check(testkit.Rows("{\"metadata\": {\"attribute\": \"test\"}, \"resource_group\": \"default\"}")) + Check(testkit.Rows("{\"metadata\": {\"attribute\": \"test\"}}")) // Create a user and specify FAILED_LOGIN_ATTEMPTS, PASSWORD_LOCK_TIME, and COMMENT. // Check the user_attributes value after alter user. @@ -1074,7 +1074,7 @@ func TestFailedLoginTrackingAlterUser(t *testing.T) { checkUserUserAttributes(tk, "testu8", "localhost", "1 1 ") tk.MustExec("alter user 'testu8'@'localhost' FAILED_LOGIN_ATTEMPTS 0 PASSWORD_LOCK_TIME 0") tk.MustQuery("select user_attributes from mysql.user where user= 'testu8' and host = 'localhost'"). - Check(testkit.Rows("{\"resource_group\": \"default\"}")) + Check(testkit.Rows("")) // Specify only FAILED_LOGIN_ATTEMPTS one attribute when creating user. // Change the value to 0 and check the user_attributes value. @@ -1082,7 +1082,7 @@ func TestFailedLoginTrackingAlterUser(t *testing.T) { tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking.failed_login_attempts') " + "from mysql.user where user='testu9' and host ='localhost'").Check(testkit.Rows("1")) tk.MustExec("ALTER USER 'testu9'@'localhost' FAILED_LOGIN_ATTEMPTS 0") - tk.MustQuery("select user_attributes from mysql.user where user='testu9' and host ='localhost'").Check(testkit.Rows("{\"resource_group\": \"default\"}")) + tk.MustQuery("select user_attributes from mysql.user where user='testu9' and host ='localhost'").Check(testkit.Rows("")) // Specify only PASSWORD_LOCK_TIME one attribute when creating user. // Change the value to 0 and check the user_attributes value. @@ -1090,7 +1090,7 @@ func TestFailedLoginTrackingAlterUser(t *testing.T) { tk.MustQuery("select JSON_EXTRACT(user_attributes, '$.Password_locking.password_lock_time_days') " + "from mysql.user where user='testu10' and host ='localhost'").Check(testkit.Rows("1")) tk.MustExec("ALTER USER 'testu10'@'localhost' PASSWORD_LOCK_TIME 0") - tk.MustQuery("select user_attributes from mysql.user where user='testu10' and host ='localhost'").Check(testkit.Rows("{\"resource_group\": \"default\"}")) + tk.MustQuery("select user_attributes from mysql.user where user='testu10' and host ='localhost'").Check(testkit.Rows("")) // Specify FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME attributes when creating user , // change the values of the two attributes to 0, and check the value of user_attributes. @@ -1104,7 +1104,7 @@ func TestFailedLoginTrackingAlterUser(t *testing.T) { "from mysql.user where user='testu11' and host ='localhost'").Check(testkit.Rows("1 0")) tk.MustExec("ALTER USER 'testu11'@'localhost' FAILED_LOGIN_ATTEMPTS 0") tk.MustQuery("select user_attributes " + - "from mysql.user where user='testu11' and host ='localhost'").Check(testkit.Rows("{\"resource_group\": \"default\"}")) + "from mysql.user where user='testu11' and host ='localhost'").Check(testkit.Rows("")) rootTK := testkit.NewTestKit(t, store) sql := new(strings.Builder) @@ -1165,7 +1165,7 @@ func TestFailedLoginTrackingCheckPrivilges(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) createAndCheck(tk, "CREATE USER 'testu1'@'localhost' IDENTIFIED BY '' FAILED_LOGIN_ATTEMPTS 1 PASSWORD_LOCK_TIME 1", - "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 1}, \"resource_group\": \"default\"}", "testu1") + "{\"Password_locking\": {\"failed_login_attempts\": 1, \"password_lock_time_days\": 1}}", "testu1") require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "testu1", Hostname: "localhost"}, nil, nil)) // Specify FAILED_LOGIN_ATTEMPTS and PASSWORD_LOCK_TIME attributes when creating user , // Check user privileges after successful login. @@ -1192,7 +1192,7 @@ func TestUserPassword(t *testing.T) { "CREATE USER 'u1'@'localhost' IDENTIFIED BY '!@#HASHhs123' FAILED_LOGIN_ATTEMPTS 3 PASSWORD_LOCK_TIME 4;", "u1", "localhost", - "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 4}, \"resource_group\": \"default\"}", + "{\"Password_locking\": {\"failed_login_attempts\": 3, \"password_lock_time_days\": 4}}", "qwe123", "!@#HASHhs123", }, @@ -1201,7 +1201,7 @@ func TestUserPassword(t *testing.T) { `CREATE USER 'u2'@'localhost' IDENTIFIED BY '!@#HASHhs123' FAILED_LOGIN_ATTEMPTS 4 PASSWORD_LOCK_TIME 3 COMMENT 'Some statements to test create user'`, "u2", "localhost", - "{\"Password_locking\": {\"failed_login_attempts\": 4, \"password_lock_time_days\": 3}, \"metadata\": {\"comment\": \"Some statements to test create user\"}, \"resource_group\": \"default\"}", + "{\"Password_locking\": {\"failed_login_attempts\": 4, \"password_lock_time_days\": 3}, \"metadata\": {\"comment\": \"Some statements to test create user\"}}", "qwe123", "!@#HASHhs123", }, diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index f08ad0c82a2b4..1c983c02945ba 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -161,7 +161,8 @@ func TestInfoSchemaFieldValue(t *testing.T) { " `DIGEST` varchar(64) DEFAULT '',\n" + " `MEM` bigint(21) unsigned DEFAULT NULL,\n" + " `DISK` bigint(21) unsigned DEFAULT NULL,\n" + - " `TxnStart` varchar(64) NOT NULL DEFAULT ''\n" + + " `TxnStart` varchar(64) NOT NULL DEFAULT '',\n" + + " `RESOURCE_GROUP_NAME` varchar(32) NOT NULL DEFAULT ''\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")) tk.MustQuery("show create table information_schema.cluster_log").Check( testkit.Rows("" + diff --git a/util/gpool/gpool.go b/util/gpool/gpool.go index bd65eaca9f505..dce4eee536165 100644 --- a/util/gpool/gpool.go +++ b/util/gpool/gpool.go @@ -46,8 +46,8 @@ var ( // BasePool is base class of pool type BasePool struct { - name string lastTuneTs atomicutil.Time + name string generator atomic.Uint64 } diff --git a/util/gpool/spmc/spmcpool.go b/util/gpool/spmc/spmcpool.go index 6f65ca98aba01..561c6f32a1f17 100644 --- a/util/gpool/spmc/spmcpool.go +++ b/util/gpool/spmc/spmcpool.go @@ -38,23 +38,22 @@ import ( // TF is the type of the context getter. It is used to get a context. // if we don't need to use CT/TF, we can define CT as any and TF as NilContext. type Pool[T any, U any, C any, CT any, TF pooltask.Context[CT]] struct { + workerCache sync.Pool + lock sync.Locker + stopCh chan struct{} + consumerFunc func(T, C, CT) U + cond *sync.Cond + taskCh chan *pooltask.TaskBox[T, U, C, CT, TF] + workers *loopQueue[T, U, C, CT, TF] + options *Options gpool.BasePool - workerCache sync.Pool - workers *loopQueue[T, U, C, CT, TF] - lock sync.Locker - cond *sync.Cond - taskCh chan *pooltask.TaskBox[T, U, C, CT, TF] taskManager pooltask.TaskManager[T, U, C, CT, TF] - options *Options - stopCh chan struct{} - consumerFunc func(T, C, CT) U + waitingTask atomicutil.Uint32 capacity atomic.Int32 running atomic.Int32 state atomic.Int32 - waiting atomic.Int32 // waiting is the number of goroutines that are waiting for the pool to be available. + waiting atomic.Int32 heartbeatDone atomic.Bool - - waitingTask atomicutil.Uint32 // waitingTask is the number of tasks that are waiting for the pool to be available. } // NewSPMCPool create a single producer, multiple consumer goroutine pool. From 02632e3858b9c283175abf6e538210839b6daf53 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Sat, 28 Jan 2023 19:08:34 +0800 Subject: [PATCH 06/24] *: add comment back Signed-off-by: BornChanger --- util/gpool/spmc/spmcpool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/gpool/spmc/spmcpool.go b/util/gpool/spmc/spmcpool.go index 561c6f32a1f17..ec236c06e94d9 100644 --- a/util/gpool/spmc/spmcpool.go +++ b/util/gpool/spmc/spmcpool.go @@ -52,7 +52,7 @@ type Pool[T any, U any, C any, CT any, TF pooltask.Context[CT]] struct { capacity atomic.Int32 running atomic.Int32 state atomic.Int32 - waiting atomic.Int32 + waiting atomic.Int32 // waiting is the number of goroutines that are waiting for the pool to be available. heartbeatDone atomic.Bool } From 18392f861eb34d756fe21ebc7c8149a426d816db Mon Sep 17 00:00:00 2001 From: BornChanger Date: Sat, 28 Jan 2023 23:39:12 +0800 Subject: [PATCH 07/24] *: polish some error message Signed-off-by: BornChanger --- ddl/resource_group_test.go | 2 ++ executor/simple.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index 3607f9503da96..79bc757a8b3c7 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -154,8 +154,10 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustQuery("select count(*) from information_schema.resource_groups").Check(testkit.Rows("2")) tk.MustGetErrCode("create user usr_fail resource group nil_group", mysql.ErrResourceGroupNotExists) + tk.MustContainErrMsg("create user usr_fail resource group nil_group", "Unknown resource group 'nil_group'") tk.MustExec("create user user2") tk.MustGetErrCode("alter user user2 resource group nil_group", mysql.ErrResourceGroupNotExists) + tk.MustContainErrMsg("alter user user2 resource group nil_group", "Unknown resource group 'nil_group'") tk.MustExec("create resource group do_not_delete_rg ru_per_sec=100") tk.MustExec("create user usr3 resource group do_not_delete_rg") diff --git a/executor/simple.go b/executor/simple.go index 94394571b9599..cc07b7a99ae88 100644 --- a/executor/simple.go +++ b/executor/simple.go @@ -1102,7 +1102,7 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm if resourceGroupName != "default" && resourceGroupName != "" { _, exists := e.is.ResourceGroupByName(model.NewCIStr(resourceGroupName)) if !exists { - return infoschema.ErrResourceGroupNotExists + return infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(resourceGroupName) } } userAttributes = append(userAttributes, fmt.Sprintf("\"resource_group\": \"%s\"", resourceGroupName)) @@ -1912,7 +1912,7 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt) if s.ResourceGroupNameOption.Value != "default" && s.ResourceGroupNameOption.Value != "" { _, exists := e.is.ResourceGroupByName(model.NewCIStr(s.ResourceGroupNameOption.Value)) if !exists { - return infoschema.ErrResourceGroupNotExists + return infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(s.ResourceGroupNameOption.Value) } } From b0929a8388af7ca2f0e6bc3e0b8c065c10a04c69 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Mon, 30 Jan 2023 14:36:00 +0800 Subject: [PATCH 08/24] *: make resource group name case insensitive Signed-off-by: BornChanger --- ddl/resource_group_test.go | 1 + executor/simple.go | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index 79bc757a8b3c7..055eb14e8256b 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -164,6 +164,7 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "do_not_delete_rg"}`)) tk.MustContainErrMsg("drop resource group do_not_delete_rg", "user [usr3] depends on the resource group to drop") tk.MustExec("alter user usr3 resource group `default`") + tk.MustExec("alter user usr3 resource group `DeFault`") tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "default"}`)) } diff --git a/executor/simple.go b/executor/simple.go index cc07b7a99ae88..e291e42e04224 100644 --- a/executor/simple.go +++ b/executor/simple.go @@ -1096,7 +1096,7 @@ func (e *SimpleExec) executeCreateUser(ctx context.Context, s *ast.CreateUserStm return infoschema.ErrResourceGroupSupportDisabled } - resourceGroupName := s.ResourceGroupNameOption.Value + resourceGroupName := strings.ToLower(s.ResourceGroupNameOption.Value) // check if specified resource group exists if resourceGroupName != "default" && resourceGroupName != "" { @@ -1909,14 +1909,15 @@ func (e *SimpleExec) executeAlterUser(ctx context.Context, s *ast.AlterUserStmt) } // check if specified resource group exists - if s.ResourceGroupNameOption.Value != "default" && s.ResourceGroupNameOption.Value != "" { - _, exists := e.is.ResourceGroupByName(model.NewCIStr(s.ResourceGroupNameOption.Value)) + resourceGroupName := strings.ToLower(s.ResourceGroupNameOption.Value) + if resourceGroupName != "default" && s.ResourceGroupNameOption.Value != "" { + _, exists := e.is.ResourceGroupByName(model.NewCIStr(resourceGroupName)) if !exists { - return infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(s.ResourceGroupNameOption.Value) + return infoschema.ErrResourceGroupNotExists.GenWithStackByArgs(resourceGroupName) } } - newAttributes = append(newAttributes, fmt.Sprintf(`"resource_group": "%s"`, s.ResourceGroupNameOption.Value)) + newAttributes = append(newAttributes, fmt.Sprintf(`"resource_group": "%s"`, resourceGroupName)) } if passwordLockingStr != "" { newAttributes = append(newAttributes, passwordLockingStr) From e55e5e0e25b2d4cdc5e851ecdce2da2193fdbb4c Mon Sep 17 00:00:00 2001 From: BornChanger Date: Mon, 30 Jan 2023 21:16:23 +0800 Subject: [PATCH 09/24] *: polish information_schema.resource_groups Signed-off-by: BornChanger --- ddl/resource_group_test.go | 45 +++++++++++-------- executor/infoschema_reader.go | 82 ++++++++++++++++++++++++++++++----- infoschema/tables.go | 14 ++++-- parser/model/model.go | 2 +- 4 files changed, 111 insertions(+), 32 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index 055eb14e8256b..5f70d148a234d 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -16,7 +16,6 @@ package ddl_test import ( "context" - "strconv" "testing" "github.com/pingcap/tidb/ddl/internal/callback" @@ -89,16 +88,16 @@ func TestResourceGroupBasic(t *testing.T) { re.Equal(uint64(2000), g.RURate) re.Equal(int64(-1), g.BurstLimit) + tk.MustQuery("select * from information_schema.resource_groups where group_name = 'x'").Check(testkit.Rows("x RU_MODE 2000 0 3000 0 ")) + tk.MustExec("drop resource group x") + g = testResourceGroupNameFromIS(t, tk.Session(), "x") + re.Nil(g) + tk.MustExec("alter resource group if exists not_exists RU_PER_SEC=2000") // Check warning message res = tk.MustQuery("show warnings") res.Check(testkit.Rows("Note 8249 Unknown resource group 'not_exists'")) - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'x' ").Check(testkit.Rows(strconv.FormatInt(g.ID, 10) + " x 2000")) - tk.MustExec("drop resource group x") - g = testResourceGroupNameFromIS(t, tk.Session(), "x") - re.Nil(g) - tk.MustExec("create resource group y " + "CPU='4000m' " + "IO_READ_BANDWIDTH='1G' " + @@ -137,20 +136,18 @@ func TestResourceGroupBasic(t *testing.T) { // Check information schema table information_schema.resource_groups tk.MustExec("create resource group x RU_PER_SEC=1000") - g1 := testResourceGroupNameFromIS(t, tk.Session(), "x") - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'x'").Check(testkit.Rows(strconv.FormatInt(g1.ID, 10) + " x 1000")) - tk.MustQuery("show create resource group x").Check(testkit.Rows("x CREATE RESOURCE GROUP `x` RU_PER_SEC=1000")) + tk.MustQuery("select * from information_schema.resource_groups where group_name = 'x'").Check(testkit.Rows("x RU_MODE 1000 0 2000 0 ")) + tk.MustQuery("show create resource group x").Check(testkit.Rows("x CREATE RESOURCE GROUP `x` RRU_PER_SEC=1000 WRU_PER_SEC=2000")) - tk.MustExec("create resource group y RU_PER_SEC=2000") - g2 := testResourceGroupNameFromIS(t, tk.Session(), "y") - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'y'").Check(testkit.Rows(strconv.FormatInt(g2.ID, 10) + " y 2000")) - tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RU_PER_SEC=2000")) + tk.MustExec("create resource group y " + + "RRU_PER_SEC=2000 " + + "WRU_PER_SEC=3000") + tk.MustQuery("select * from information_schema.resource_groups where group_name = 'y'").Check(testkit.Rows("y RU_MODE 2000 0 3000 0 ")) + tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RRU_PER_SEC=2000 WRU_PER_SEC=3000")) tk.MustExec("alter resource group y RU_PER_SEC=4000") - - g2 = testResourceGroupNameFromIS(t, tk.Session(), "y") - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'y'").Check(testkit.Rows(strconv.FormatInt(g2.ID, 10) + " y 4000")) - tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RU_PER_SEC=4000")) + tk.MustQuery("select * from information_schema.resource_groups where group_name = 'y'").Check(testkit.Rows("y RU_MODE 4000 0 2000 0 ")) + tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RRU_PER_SEC=4000 WRU_PER_SEC=2000")) tk.MustQuery("select count(*) from information_schema.resource_groups").Check(testkit.Rows("2")) tk.MustGetErrCode("create user usr_fail resource group nil_group", mysql.ErrResourceGroupNotExists) @@ -159,6 +156,20 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustGetErrCode("alter user user2 resource group nil_group", mysql.ErrResourceGroupNotExists) tk.MustContainErrMsg("alter user user2 resource group nil_group", "Unknown resource group 'nil_group'") + /* RAW_MODE is disabled at this time + tk.MustExec("create resource group z " + + "CPU='4000m' " + + "IO_READ_BANDWIDTH='1G' " + + "IO_WRITE_BANDWIDTH='300M'") + tk.MustQuery("select * from information_schema.resource_groups where group_name = 'z'").Check(testkit.Rows("z RAW_MODE 4000 1000000000 300000000")) + tk.MustQuery("show create resource group z").Check(testkit.Rows("z CREATE RESOURCE GROUP `z` CPU=\"4000m\" IO_READ_BANDWIDTH=\"1G\" IO_WRITE_BANDWIDTH=\"300M\"")) + */ + + tk.MustContainErrMsg("create resource group z "+ + "CPU='4000m' "+ + "IO_READ_BANDWIDTH='1G' "+ + "IO_WRITE_BANDWIDTH='300M'", resourcegroup.ErrInvalidGroupSettings) + tk.MustExec("create resource group do_not_delete_rg ru_per_sec=100") tk.MustExec("create user usr3 resource group do_not_delete_rg") tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "do_not_delete_rg"}`)) diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index d0b3afa4e5364..0b94ab8f25b20 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/deadlock" + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/tidb/ddl/label" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/domain/infosync" @@ -185,7 +186,7 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex case infoschema.ClusterTableMemoryUsageOpsHistory: err = e.setDataForClusterMemoryUsageOpsHistory(sctx) case infoschema.TableResourceGroups: - err = e.setDataFromResourceGroups(sctx) + err = e.setDataFromResourceGroups() } if err != nil { return nil, err @@ -3388,17 +3389,78 @@ func (e *memtableRetriever) setDataFromPlacementPolicies(sctx sessionctx.Context return nil } -func (e *memtableRetriever) setDataFromResourceGroups(sctx sessionctx.Context) error { - is := sessiontxn.GetTxnManager(sctx).GetTxnInfoSchema() - resourceGroups := is.AllResourceGroups() +func (e *memtableRetriever) setDataFromResourceGroups() error { + resourceGroups, err := infosync.GetAllResourceGroups(context.TODO()) + if err != nil { + return errors.Errorf("failed to access resource group manager, error message is %s", err.Error()) + } rows := make([][]types.Datum, 0, len(resourceGroups)) for _, group := range resourceGroups { - row := types.MakeDatums( - group.ID, - group.Name.O, - group.RURate, - ) - rows = append(rows, row) + mode := "" + switch group.Mode { + case rmpb.GroupMode_RUMode: + mode = "RU_MODE" + rru_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.RRU.Settings.String(), ":")[1], " ")) + if err != nil { + return errors.Errorf("invalid fill rate of RRU for resource group %s", group.Name) + } + wru_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.WRU.Settings.String(), ":")[1], " ")) + if err != nil { + return errors.Errorf("invalid fill rate of WRU for resource group %s", group.Name) + } + row := types.MakeDatums( + group.Name, + mode, + rru_setting, + int(group.RUSettings.RRU.Tokens), + wru_setting, + int(group.RUSettings.WRU.Tokens), + nil, + nil, + nil, + ) + rows = append(rows, row) + case rmpb.GroupMode_RawMode: + mode = "RAW_MODE" + cpu_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.Cpu.Settings.String(), ":")[1], " ")) + if err != nil { + return errors.Errorf("invalid fill rate of CPU for resource group %s", group.Name) + } + read_io_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoRead.Settings.String(), ":")[1], " ")) + if err != nil { + return errors.Errorf("invalid fill rate of READ for resource group %s", group.Name) + } + write_io_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoWrite.Settings.String(), ":")[1], " ")) + if err != nil { + return errors.Errorf("invalid fill rate of WRITE for resource group %s", group.Name) + } + row := types.MakeDatums( + group.Name, + mode, + nil, + nil, + nil, + nil, + cpu_setting, + read_io_setting, + write_io_setting, + ) + rows = append(rows, row) + default: + mode = "UNKNOWN_MODE" + row := types.MakeDatums( + group.Name, + mode, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + ) + rows = append(rows, row) + } } e.rows = rows return nil diff --git a/infoschema/tables.go b/infoschema/tables.go index e262da273cf3a..2285b463efab9 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1593,10 +1593,16 @@ var tableMemoryUsageOpsHistoryCols = []columnInfo{ } var tableResourceGroupsCols = []columnInfo{ - {name: "GROUP_ID", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, - {name: "GROUP_NAME", tp: mysql.TypeVarchar, size: 512, flag: mysql.NotNullFlag}, - {name: "RU_PER_SECOND", tp: mysql.TypeLonglong, size: 64}, - // {name: "BURSTABLE", tp: mysql.TypeVarchar, size: 10}, + {name: "GROUP_NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag}, + {name: "MODE", tp: mysql.TypeVarchar, size: 12, flag: mysql.NotNullFlag}, + {name: "RRU", tp: mysql.TypeLonglong, size: 21}, + {name: "RRU_TOKEN", tp: mysql.TypeLonglong, size: 21}, + {name: "WRU", tp: mysql.TypeLonglong, size: 21}, + {name: "WRU_TOKEN", tp: mysql.TypeLonglong, size: 21}, + {name: "CPU", tp: mysql.TypeLonglong, size: 21}, + {name: "READ_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, + {name: "WRITE_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, + {name: "BURSTABLE", tp: mysql.TypeVarchar, size: 5}, } // GetShardingInfo returns a nil or description string for the sharding information of given TableInfo. diff --git a/parser/model/model.go b/parser/model/model.go index b154ef4dac654..b953b1140bf7b 100644 --- a/parser/model/model.go +++ b/parser/model/model.go @@ -1880,7 +1880,7 @@ func (p *ResourceGroupSettings) Clone() *ResourceGroupSettings { return &cloned } -// ResourceGroupInfo is the struct to store the placement policy. +// ResourceGroupInfo is the struct to store the resource group. type ResourceGroupInfo struct { *ResourceGroupSettings ID int64 `json:"id"` From cebbd6ed3798a3ab9e630e7a990274dd97e4660d Mon Sep 17 00:00:00 2001 From: BornChanger Date: Mon, 30 Jan 2023 21:54:18 +0800 Subject: [PATCH 10/24] *: code format Signed-off-by: BornChanger --- executor/infoschema_reader.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 0b94ab8f25b20..0080741e440f6 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -3400,20 +3400,20 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { switch group.Mode { case rmpb.GroupMode_RUMode: mode = "RU_MODE" - rru_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.RRU.Settings.String(), ":")[1], " ")) + rruSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.RRU.Settings.String(), ":")[1], " ")) if err != nil { return errors.Errorf("invalid fill rate of RRU for resource group %s", group.Name) } - wru_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.WRU.Settings.String(), ":")[1], " ")) + wruSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.WRU.Settings.String(), ":")[1], " ")) if err != nil { return errors.Errorf("invalid fill rate of WRU for resource group %s", group.Name) } row := types.MakeDatums( group.Name, mode, - rru_setting, + rruSetting, int(group.RUSettings.RRU.Tokens), - wru_setting, + wruSetting, int(group.RUSettings.WRU.Tokens), nil, nil, @@ -3422,15 +3422,15 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { rows = append(rows, row) case rmpb.GroupMode_RawMode: mode = "RAW_MODE" - cpu_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.Cpu.Settings.String(), ":")[1], " ")) + cpuSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.Cpu.Settings.String(), ":")[1], " ")) if err != nil { return errors.Errorf("invalid fill rate of CPU for resource group %s", group.Name) } - read_io_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoRead.Settings.String(), ":")[1], " ")) + readIoSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoRead.Settings.String(), ":")[1], " ")) if err != nil { return errors.Errorf("invalid fill rate of READ for resource group %s", group.Name) } - write_io_setting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoWrite.Settings.String(), ":")[1], " ")) + writeIoSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoWrite.Settings.String(), ":")[1], " ")) if err != nil { return errors.Errorf("invalid fill rate of WRITE for resource group %s", group.Name) } @@ -3441,9 +3441,9 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { nil, nil, nil, - cpu_setting, - read_io_setting, - write_io_setting, + cpuSetting, + readIoSetting, + writeIoSetting, ) rows = append(rows, row) default: From f71707fd4769287060a6c747702e1f9a9a50eb30 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Mon, 30 Jan 2023 23:05:55 +0800 Subject: [PATCH 11/24] *: make bazel happy Signed-off-by: BornChanger --- executor/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/executor/BUILD.bazel b/executor/BUILD.bazel index c8881bec70917..4eaeffdfb3c4f 100644 --- a/executor/BUILD.bazel +++ b/executor/BUILD.bazel @@ -214,6 +214,7 @@ go_library( "@com_github_pingcap_kvproto//pkg/encryptionpb", "@com_github_pingcap_kvproto//pkg/kvrpcpb", "@com_github_pingcap_kvproto//pkg/metapb", + "@com_github_pingcap_kvproto//pkg/resource_manager", "@com_github_pingcap_kvproto//pkg/tikvpb", "@com_github_pingcap_log//:log", "@com_github_pingcap_sysutil//:sysutil", From 5c345b13a66e34286cd3f89509fbf7b5bd04b268 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Tue, 31 Jan 2023 10:53:46 +0800 Subject: [PATCH 12/24] *: rename column Signed-off-by: BornChanger --- infoschema/tables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infoschema/tables.go b/infoschema/tables.go index 2285b463efab9..3e0fa888d4cd6 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1593,7 +1593,7 @@ var tableMemoryUsageOpsHistoryCols = []columnInfo{ } var tableResourceGroupsCols = []columnInfo{ - {name: "GROUP_NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag}, + {name: "NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag}, {name: "MODE", tp: mysql.TypeVarchar, size: 12, flag: mysql.NotNullFlag}, {name: "RRU", tp: mysql.TypeLonglong, size: 21}, {name: "RRU_TOKEN", tp: mysql.TypeLonglong, size: 21}, From fa80bae9daa126b7ae02f9a2987f9a6709ad871f Mon Sep 17 00:00:00 2001 From: BornChanger Date: Tue, 31 Jan 2023 15:50:35 +0800 Subject: [PATCH 13/24] *: fix issue #40854 Signed-off-by: BornChanger --- domain/domain_sysvars.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/domain/domain_sysvars.go b/domain/domain_sysvars.go index 6988cedcc9b52..982ca8f06af58 100644 --- a/domain/domain_sysvars.go +++ b/domain/domain_sysvars.go @@ -76,7 +76,10 @@ func (do *Domain) setGlobalResourceControl(enable bool) { } else { variable.DisableGlobalResourceControlFunc() } - logutil.BgLogger().Info("set resource control", zap.Bool("enable", enable)) + + if enable != variable.EnableResourceControl.Load() { + logutil.BgLogger().Info("set resource control", zap.Bool("enable", enable)) + } } // updatePDClient is used to set the dynamic option into the PD client. From 3107505fa7f11d3eff4c967604db81586cefeb2a Mon Sep 17 00:00:00 2001 From: BornChanger Date: Tue, 31 Jan 2023 17:59:26 +0800 Subject: [PATCH 14/24] *: address comments Signed-off-by: BornChanger --- domain/domain_sysvars.go | 4 ---- sessionctx/variable/sysvar.go | 9 +++++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/domain/domain_sysvars.go b/domain/domain_sysvars.go index 982ca8f06af58..3d102687454d2 100644 --- a/domain/domain_sysvars.go +++ b/domain/domain_sysvars.go @@ -76,10 +76,6 @@ func (do *Domain) setGlobalResourceControl(enable bool) { } else { variable.DisableGlobalResourceControlFunc() } - - if enable != variable.EnableResourceControl.Load() { - logutil.BgLogger().Info("set resource control", zap.Bool("enable", enable)) - } } // updatePDClient is used to set the dynamic option into the PD client. diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index d9f8f9407b7fe..70afdf4ae8c72 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "go.uber.org/zap" "math" "runtime" "strconv" @@ -2322,8 +2323,12 @@ var defaultSysVars = []*SysVar{ }, }, {Scope: ScopeGlobal, Name: TiDBEnableResourceControl, Value: BoolToOnOff(DefTiDBEnableResourceControl), Type: TypeBool, SetGlobal: func(ctx context.Context, vars *SessionVars, s string) error { - EnableResourceControl.Store(TiDBOptOn(s)) - (*SetGlobalResourceControl.Load())(TiDBOptOn(s)) + if TiDBOptOn(s) != EnableResourceControl.Load() { + EnableResourceControl.Store(TiDBOptOn(s)) + (*SetGlobalResourceControl.Load())(TiDBOptOn(s)) + logutil.BgLogger().Info("set resource control", zap.Bool("enable", TiDBOptOn(s))) + TiDBOptOn(s) + } return nil }, GetGlobal: func(ctx context.Context, vars *SessionVars) (string, error) { return BoolToOnOff(EnableResourceControl.Load()), nil From a49053f2a3bbea5b6e9e0d77e3eb2e572d3ab0fa Mon Sep 17 00:00:00 2001 From: BornChanger Date: Tue, 31 Jan 2023 18:27:30 +0800 Subject: [PATCH 15/24] *: code format Signed-off-by: BornChanger --- sessionctx/variable/BUILD.bazel | 1 + sessionctx/variable/sysvar.go | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sessionctx/variable/BUILD.bazel b/sessionctx/variable/BUILD.bazel index b178ccf0a95da..c1d6e0b8b9651 100644 --- a/sessionctx/variable/BUILD.bazel +++ b/sessionctx/variable/BUILD.bazel @@ -68,6 +68,7 @@ go_library( "@org_golang_x_exp//maps", "@org_golang_x_exp//slices", "@org_uber_go_atomic//:atomic", + "@org_uber_go_zap//:zap", ], ) diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 70afdf4ae8c72..3da39a2ef80e9 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "fmt" - "go.uber.org/zap" "math" "runtime" "strconv" @@ -49,6 +48,7 @@ import ( tikvcfg "github.com/tikv/client-go/v2/config" tikvstore "github.com/tikv/client-go/v2/kv" atomic2 "go.uber.org/atomic" + "go.uber.org/zap" ) // All system variables declared here are ordered by their scopes, which follow the order of scopes below: @@ -2327,7 +2327,6 @@ var defaultSysVars = []*SysVar{ EnableResourceControl.Store(TiDBOptOn(s)) (*SetGlobalResourceControl.Load())(TiDBOptOn(s)) logutil.BgLogger().Info("set resource control", zap.Bool("enable", TiDBOptOn(s))) - TiDBOptOn(s) } return nil }, GetGlobal: func(ctx context.Context, vars *SessionVars) (string, error) { From 62d39c4bd53c0b3866380b3b54dda35a5d03b4bf Mon Sep 17 00:00:00 2001 From: BornChanger Date: Tue, 31 Jan 2023 22:25:39 +0800 Subject: [PATCH 16/24] *: fix case Signed-off-by: BornChanger --- ddl/resource_group_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index 5f70d148a234d..749e5f2f1c2e8 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -88,7 +88,8 @@ func TestResourceGroupBasic(t *testing.T) { re.Equal(uint64(2000), g.RURate) re.Equal(int64(-1), g.BurstLimit) - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'x'").Check(testkit.Rows("x RU_MODE 2000 0 3000 0 ")) + tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x RU_MODE 2000 0 3000 0 ")) + tk.MustExec("drop resource group x") g = testResourceGroupNameFromIS(t, tk.Session(), "x") re.Nil(g) @@ -142,7 +143,7 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustExec("create resource group y " + "RRU_PER_SEC=2000 " + "WRU_PER_SEC=3000") - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'y'").Check(testkit.Rows("y RU_MODE 2000 0 3000 0 ")) + tk.MustQuery("select * from information_schema.resource_groups where name = 'y'").Check(testkit.Rows("y RU_MODE 2000 0 3000 0 ")) tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RRU_PER_SEC=2000 WRU_PER_SEC=3000")) tk.MustExec("alter resource group y RU_PER_SEC=4000") @@ -161,7 +162,7 @@ func TestResourceGroupBasic(t *testing.T) { "CPU='4000m' " + "IO_READ_BANDWIDTH='1G' " + "IO_WRITE_BANDWIDTH='300M'") - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'z'").Check(testkit.Rows("z RAW_MODE 4000 1000000000 300000000")) + tk.MustQuery("select * from information_schema.resource_groups where name = 'z'").Check(testkit.Rows("z RAW_MODE 4000 1000000000 300000000")) tk.MustQuery("show create resource group z").Check(testkit.Rows("z CREATE RESOURCE GROUP `z` CPU=\"4000m\" IO_READ_BANDWIDTH=\"1G\" IO_WRITE_BANDWIDTH=\"300M\"")) */ From 04c03e7c469f753fdc64b419bd98e470b0368991 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Thu, 2 Feb 2023 18:53:15 +0800 Subject: [PATCH 17/24] *: rebase Signed-off-by: BornChanger --- executor/infoschema_reader.go | 98 ++++++++++++++++++----------------- infoschema/tables.go | 16 +++--- 2 files changed, 58 insertions(+), 56 deletions(-) diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 0080741e440f6..d7cc521271275 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -3396,65 +3396,69 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { } rows := make([][]types.Datum, 0, len(resourceGroups)) for _, group := range resourceGroups { - mode := "" + //mode := "" switch group.Mode { case rmpb.GroupMode_RUMode: - mode = "RU_MODE" - rruSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.RRU.Settings.String(), ":")[1], " ")) + //mode = "RU_MODE" + ruSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.RU.Settings.String(), ":")[1], " ")) if err != nil { - return errors.Errorf("invalid fill rate of RRU for resource group %s", group.Name) + return errors.Errorf("invalid fill rate of RU for resource group %s", group.Name) } - wruSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.WRU.Settings.String(), ":")[1], " ")) - if err != nil { - return errors.Errorf("invalid fill rate of WRU for resource group %s", group.Name) + burstable := false + if group.RUSettings.RU.GetSettings().BurstLimit < 0 { + burstable = true } row := types.MakeDatums( group.Name, - mode, - rruSetting, - int(group.RUSettings.RRU.Tokens), - wruSetting, - int(group.RUSettings.WRU.Tokens), - nil, - nil, - nil, - ) - rows = append(rows, row) - case rmpb.GroupMode_RawMode: - mode = "RAW_MODE" - cpuSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.Cpu.Settings.String(), ":")[1], " ")) - if err != nil { - return errors.Errorf("invalid fill rate of CPU for resource group %s", group.Name) - } - readIoSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoRead.Settings.String(), ":")[1], " ")) - if err != nil { - return errors.Errorf("invalid fill rate of READ for resource group %s", group.Name) - } - writeIoSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoWrite.Settings.String(), ":")[1], " ")) - if err != nil { - return errors.Errorf("invalid fill rate of WRITE for resource group %s", group.Name) - } - row := types.MakeDatums( - group.Name, - mode, - nil, - nil, - nil, - nil, - cpuSetting, - readIoSetting, - writeIoSetting, + //mode, + ruSetting, + int(group.RUSettings.RU.Tokens), + //nil, + //nil, + //nil, + burstable, ) rows = append(rows, row) + /* + case rmpb.GroupMode_RawMode: + mode = "RAW_MODE" + cpuSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.Cpu.Settings.String(), ":")[1], " ")) + if err != nil { + return errors.Errorf("invalid fill rate of CPU for resource group %s", group.Name) + } + readIoSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoRead.Settings.String(), ":")[1], " ")) + if err != nil { + return errors.Errorf("invalid fill rate of READ for resource group %s", group.Name) + } + writeIoSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoWrite.Settings.String(), ":")[1], " ")) + if err != nil { + return errors.Errorf("invalid fill rate of WRITE for resource group %s", group.Name) + } + + burstable := false + if group.RawResourceSettings. < 0 { + burstable = true + } + row := types.MakeDatums( + group.Name, + mode, + nil, + nil, + cpuSetting, + readIoSetting, + writeIoSetting, + burstable, + ) + rows = append(rows, row) + */ default: - mode = "UNKNOWN_MODE" + //mode = "UNKNOWN_MODE" row := types.MakeDatums( group.Name, - mode, - nil, - nil, - nil, - nil, + //mode, + //nil, + //nil, + //nil, nil, nil, nil, diff --git a/infoschema/tables.go b/infoschema/tables.go index 3e0fa888d4cd6..70b35fea92ae0 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1594,15 +1594,13 @@ var tableMemoryUsageOpsHistoryCols = []columnInfo{ var tableResourceGroupsCols = []columnInfo{ {name: "NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag}, - {name: "MODE", tp: mysql.TypeVarchar, size: 12, flag: mysql.NotNullFlag}, - {name: "RRU", tp: mysql.TypeLonglong, size: 21}, - {name: "RRU_TOKEN", tp: mysql.TypeLonglong, size: 21}, - {name: "WRU", tp: mysql.TypeLonglong, size: 21}, - {name: "WRU_TOKEN", tp: mysql.TypeLonglong, size: 21}, - {name: "CPU", tp: mysql.TypeLonglong, size: 21}, - {name: "READ_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, - {name: "WRITE_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, - {name: "BURSTABLE", tp: mysql.TypeVarchar, size: 5}, + //{name: "MODE", tp: mysql.TypeVarchar, size: 12, flag: mysql.NotNullFlag}, + {name: "RU", tp: mysql.TypeLonglong, size: 21}, + {name: "RU_TOKEN", tp: mysql.TypeLonglong, size: 21}, + //{name: "CPU", tp: mysql.TypeLonglong, size: 21}, + //{name: "READ_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, + //{name: "WRITE_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, + {name: "BURSTABLE", tp: mysql.TypeBit}, } // GetShardingInfo returns a nil or description string for the sharding information of given TableInfo. From 95cf62d61325c1abe29a421b3a8a2cdca31bbfc6 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 3 Feb 2023 12:06:49 +0800 Subject: [PATCH 18/24] *: tune some code Signed-off-by: BornChanger --- ddl/resource_group_test.go | 4 +--- executor/infoschema_reader.go | 6 +++--- infoschema/tables.go | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index 749e5f2f1c2e8..c477ba653452e 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -140,9 +140,7 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustQuery("select * from information_schema.resource_groups where group_name = 'x'").Check(testkit.Rows("x RU_MODE 1000 0 2000 0 ")) tk.MustQuery("show create resource group x").Check(testkit.Rows("x CREATE RESOURCE GROUP `x` RRU_PER_SEC=1000 WRU_PER_SEC=2000")) - tk.MustExec("create resource group y " + - "RRU_PER_SEC=2000 " + - "WRU_PER_SEC=3000") + tk.MustExec("create resource group y RU_PER_SEC=2000") tk.MustQuery("select * from information_schema.resource_groups where name = 'y'").Check(testkit.Rows("y RU_MODE 2000 0 3000 0 ")) tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RRU_PER_SEC=2000 WRU_PER_SEC=3000")) diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index d7cc521271275..4298215d77792 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -3404,15 +3404,15 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { if err != nil { return errors.Errorf("invalid fill rate of RU for resource group %s", group.Name) } - burstable := false + burstable := "YES" if group.RUSettings.RU.GetSettings().BurstLimit < 0 { - burstable = true + burstable = "NO" } row := types.MakeDatums( group.Name, //mode, ruSetting, - int(group.RUSettings.RU.Tokens), + uint64(group.RUSettings.RU.Tokens), //nil, //nil, //nil, diff --git a/infoschema/tables.go b/infoschema/tables.go index 70b35fea92ae0..0e992c48afca8 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1595,12 +1595,12 @@ var tableMemoryUsageOpsHistoryCols = []columnInfo{ var tableResourceGroupsCols = []columnInfo{ {name: "NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag}, //{name: "MODE", tp: mysql.TypeVarchar, size: 12, flag: mysql.NotNullFlag}, - {name: "RU", tp: mysql.TypeLonglong, size: 21}, - {name: "RU_TOKEN", tp: mysql.TypeLonglong, size: 21}, + {name: "RU_FILLRATE", tp: mysql.TypeLonglong, size: 21}, + {name: "RU_TOKENS", tp: mysql.TypeLonglong, size: 21}, //{name: "CPU", tp: mysql.TypeLonglong, size: 21}, //{name: "READ_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, //{name: "WRITE_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, - {name: "BURSTABLE", tp: mysql.TypeBit}, + {name: "BURSTABLE", tp: mysql.TypeVarchar, size: 3}, } // GetShardingInfo returns a nil or description string for the sharding information of given TableInfo. From 8686543d9cb16b83e178738267aa0eeb77b17e8b Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 3 Feb 2023 12:22:35 +0800 Subject: [PATCH 19/24] *: fix case Signed-off-by: BornChanger --- ddl/resource_group_test.go | 21 ++++++++------------- executor/infoschema_reader.go | 12 ++++-------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index c477ba653452e..ace6ee6303aaa 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -88,7 +88,7 @@ func TestResourceGroupBasic(t *testing.T) { re.Equal(uint64(2000), g.RURate) re.Equal(int64(-1), g.BurstLimit) - tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x RU_MODE 2000 0 3000 0 ")) + tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x 2000 0 YES")) tk.MustExec("drop resource group x") g = testResourceGroupNameFromIS(t, tk.Session(), "x") @@ -137,16 +137,16 @@ func TestResourceGroupBasic(t *testing.T) { // Check information schema table information_schema.resource_groups tk.MustExec("create resource group x RU_PER_SEC=1000") - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'x'").Check(testkit.Rows("x RU_MODE 1000 0 2000 0 ")) - tk.MustQuery("show create resource group x").Check(testkit.Rows("x CREATE RESOURCE GROUP `x` RRU_PER_SEC=1000 WRU_PER_SEC=2000")) + tk.MustQuery("select * from information_schema.resource_groups where name = 'x'").Check(testkit.Rows("x 1000 0 NO")) + tk.MustQuery("show create resource group x").Check(testkit.Rows("x CREATE RESOURCE GROUP `x` RU_PER_SEC=1000")) tk.MustExec("create resource group y RU_PER_SEC=2000") - tk.MustQuery("select * from information_schema.resource_groups where name = 'y'").Check(testkit.Rows("y RU_MODE 2000 0 3000 0 ")) - tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RRU_PER_SEC=2000 WRU_PER_SEC=3000")) + tk.MustQuery("select * from information_schema.resource_groups where name = 'y'").Check(testkit.Rows("y 2000 0 NO")) + tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RU_PER_SEC=2000")) - tk.MustExec("alter resource group y RU_PER_SEC=4000") - tk.MustQuery("select * from information_schema.resource_groups where group_name = 'y'").Check(testkit.Rows("y RU_MODE 4000 0 2000 0 ")) - tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RRU_PER_SEC=4000 WRU_PER_SEC=2000")) + tk.MustExec("alter resource group y RU_PER_SEC=4000 BURSTABLE") + tk.MustQuery("select * from information_schema.resource_groups where name = 'y'").Check(testkit.Rows("y 4000 0 YES")) + tk.MustQuery("show create resource group y").Check(testkit.Rows("y CREATE RESOURCE GROUP `y` RU_PER_SEC=4000 BURSTABLE")) tk.MustQuery("select count(*) from information_schema.resource_groups").Check(testkit.Rows("2")) tk.MustGetErrCode("create user usr_fail resource group nil_group", mysql.ErrResourceGroupNotExists) @@ -164,11 +164,6 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustQuery("show create resource group z").Check(testkit.Rows("z CREATE RESOURCE GROUP `z` CPU=\"4000m\" IO_READ_BANDWIDTH=\"1G\" IO_WRITE_BANDWIDTH=\"300M\"")) */ - tk.MustContainErrMsg("create resource group z "+ - "CPU='4000m' "+ - "IO_READ_BANDWIDTH='1G' "+ - "IO_WRITE_BANDWIDTH='300M'", resourcegroup.ErrInvalidGroupSettings) - tk.MustExec("create resource group do_not_delete_rg ru_per_sec=100") tk.MustExec("create user usr3 resource group do_not_delete_rg") tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "do_not_delete_rg"}`)) diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 4298215d77792..5ea9db5924cd0 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -3397,21 +3397,17 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { rows := make([][]types.Datum, 0, len(resourceGroups)) for _, group := range resourceGroups { //mode := "" + burstable := "NO" switch group.Mode { case rmpb.GroupMode_RUMode: //mode = "RU_MODE" - ruSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RUSettings.RU.Settings.String(), ":")[1], " ")) - if err != nil { - return errors.Errorf("invalid fill rate of RU for resource group %s", group.Name) - } - burstable := "YES" - if group.RUSettings.RU.GetSettings().BurstLimit < 0 { - burstable = "NO" + if group.RUSettings.RU.Settings.BurstLimit < 0 { + burstable = "YES" } row := types.MakeDatums( group.Name, //mode, - ruSetting, + group.RUSettings.RU.Settings.FillRate, uint64(group.RUSettings.RU.Tokens), //nil, //nil, From 8c35622e24d82a429195bee16b24b6712a8b9fd0 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 3 Feb 2023 12:33:01 +0800 Subject: [PATCH 20/24] *: adjust case Signed-off-by: BornChanger --- ddl/resource_group_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index ace6ee6303aaa..59c1df5b409ad 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -155,14 +155,13 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustGetErrCode("alter user user2 resource group nil_group", mysql.ErrResourceGroupNotExists) tk.MustContainErrMsg("alter user user2 resource group nil_group", "Unknown resource group 'nil_group'") - /* RAW_MODE is disabled at this time tk.MustExec("create resource group z " + "CPU='4000m' " + "IO_READ_BANDWIDTH='1G' " + "IO_WRITE_BANDWIDTH='300M'") - tk.MustQuery("select * from information_schema.resource_groups where name = 'z'").Check(testkit.Rows("z RAW_MODE 4000 1000000000 300000000")) + // RAW_MODE is not supported by information_schema.resource_groups + //tk.MustQuery("select * from information_schema.resource_groups where name = 'z'").Check(testkit.Rows("z RAW_MODE 4000 1000000000 300000000")) tk.MustQuery("show create resource group z").Check(testkit.Rows("z CREATE RESOURCE GROUP `z` CPU=\"4000m\" IO_READ_BANDWIDTH=\"1G\" IO_WRITE_BANDWIDTH=\"300M\"")) - */ tk.MustExec("create resource group do_not_delete_rg ru_per_sec=100") tk.MustExec("create user usr3 resource group do_not_delete_rg") From 4227b27cd2a22d2031d1f8bf3133d1a867a40ef9 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 3 Feb 2023 14:28:16 +0800 Subject: [PATCH 21/24] *: rename column Signed-off-by: BornChanger --- ddl/resource_group_test.go | 1 + infoschema/tables.go | 2 +- infoschema/tables_test.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index 59c1df5b409ad..68a08409b847c 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -168,6 +168,7 @@ func TestResourceGroupBasic(t *testing.T) { tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "do_not_delete_rg"}`)) tk.MustContainErrMsg("drop resource group do_not_delete_rg", "user [usr3] depends on the resource group to drop") tk.MustExec("alter user usr3 resource group `default`") + tk.MustExec("alter user usr3 resource group ``") tk.MustExec("alter user usr3 resource group `DeFault`") tk.MustQuery("select user_attributes from mysql.user where user = 'usr3'").Check(testkit.Rows(`{"resource_group": "default"}`)) } diff --git a/infoschema/tables.go b/infoschema/tables.go index 0e992c48afca8..aca6067ff588f 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -825,7 +825,7 @@ var tableProcesslistCols = []columnInfo{ {name: "MEM", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "DISK", tp: mysql.TypeLonglong, size: 21, flag: mysql.UnsignedFlag}, {name: "TxnStart", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, deflt: ""}, - {name: "RESOURCE_GROUP_NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag, deflt: ""}, + {name: "RESOURCE_GROUP", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag, deflt: ""}, } var tableTiDBIndexesCols = []columnInfo{ diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 1c983c02945ba..d44721090b8b6 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -162,7 +162,7 @@ func TestInfoSchemaFieldValue(t *testing.T) { " `MEM` bigint(21) unsigned DEFAULT NULL,\n" + " `DISK` bigint(21) unsigned DEFAULT NULL,\n" + " `TxnStart` varchar(64) NOT NULL DEFAULT '',\n" + - " `RESOURCE_GROUP_NAME` varchar(32) NOT NULL DEFAULT ''\n" + + " `RESOURCE_GROUP` varchar(32) NOT NULL DEFAULT ''\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin")) tk.MustQuery("show create table information_schema.cluster_log").Check( testkit.Rows("" + From 07b7220a4d7ce1abde7e53e341878633001acae5 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 3 Feb 2023 16:31:05 +0800 Subject: [PATCH 22/24] *: rename column Signed-off-by: BornChanger --- infoschema/tables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infoschema/tables.go b/infoschema/tables.go index aca6067ff588f..1358bb94bf11e 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1595,7 +1595,7 @@ var tableMemoryUsageOpsHistoryCols = []columnInfo{ var tableResourceGroupsCols = []columnInfo{ {name: "NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag}, //{name: "MODE", tp: mysql.TypeVarchar, size: 12, flag: mysql.NotNullFlag}, - {name: "RU_FILLRATE", tp: mysql.TypeLonglong, size: 21}, + {name: "RU_PER_SEC", tp: mysql.TypeLonglong, size: 21}, {name: "RU_TOKENS", tp: mysql.TypeLonglong, size: 21}, //{name: "CPU", tp: mysql.TypeLonglong, size: 21}, //{name: "READ_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, From 7086b8bbd0b455ac878d26434fa4ce9c28d681f8 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 3 Feb 2023 19:24:56 +0800 Subject: [PATCH 23/24] *: clean up code Signed-off-by: BornChanger --- ddl/resource_group_test.go | 2 -- executor/infoschema_reader.go | 32 -------------------------------- infoschema/tables.go | 4 ---- 3 files changed, 38 deletions(-) diff --git a/ddl/resource_group_test.go b/ddl/resource_group_test.go index 68a08409b847c..a0921037c475d 100644 --- a/ddl/resource_group_test.go +++ b/ddl/resource_group_test.go @@ -159,8 +159,6 @@ func TestResourceGroupBasic(t *testing.T) { "CPU='4000m' " + "IO_READ_BANDWIDTH='1G' " + "IO_WRITE_BANDWIDTH='300M'") - // RAW_MODE is not supported by information_schema.resource_groups - //tk.MustQuery("select * from information_schema.resource_groups where name = 'z'").Check(testkit.Rows("z RAW_MODE 4000 1000000000 300000000")) tk.MustQuery("show create resource group z").Check(testkit.Rows("z CREATE RESOURCE GROUP `z` CPU=\"4000m\" IO_READ_BANDWIDTH=\"1G\" IO_WRITE_BANDWIDTH=\"300M\"")) tk.MustExec("create resource group do_not_delete_rg ru_per_sec=100") diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 5ea9db5924cd0..29d3896a2c72c 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -3415,38 +3415,6 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { burstable, ) rows = append(rows, row) - /* - case rmpb.GroupMode_RawMode: - mode = "RAW_MODE" - cpuSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.Cpu.Settings.String(), ":")[1], " ")) - if err != nil { - return errors.Errorf("invalid fill rate of CPU for resource group %s", group.Name) - } - readIoSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoRead.Settings.String(), ":")[1], " ")) - if err != nil { - return errors.Errorf("invalid fill rate of READ for resource group %s", group.Name) - } - writeIoSetting, err := strconv.Atoi(strings.Trim(strings.Split(group.RawResourceSettings.IoWrite.Settings.String(), ":")[1], " ")) - if err != nil { - return errors.Errorf("invalid fill rate of WRITE for resource group %s", group.Name) - } - - burstable := false - if group.RawResourceSettings. < 0 { - burstable = true - } - row := types.MakeDatums( - group.Name, - mode, - nil, - nil, - cpuSetting, - readIoSetting, - writeIoSetting, - burstable, - ) - rows = append(rows, row) - */ default: //mode = "UNKNOWN_MODE" row := types.MakeDatums( diff --git a/infoschema/tables.go b/infoschema/tables.go index 1358bb94bf11e..3cbf9ee1b464f 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1594,12 +1594,8 @@ var tableMemoryUsageOpsHistoryCols = []columnInfo{ var tableResourceGroupsCols = []columnInfo{ {name: "NAME", tp: mysql.TypeVarchar, size: resourcegroup.MaxGroupNameLength, flag: mysql.NotNullFlag}, - //{name: "MODE", tp: mysql.TypeVarchar, size: 12, flag: mysql.NotNullFlag}, {name: "RU_PER_SEC", tp: mysql.TypeLonglong, size: 21}, {name: "RU_TOKENS", tp: mysql.TypeLonglong, size: 21}, - //{name: "CPU", tp: mysql.TypeLonglong, size: 21}, - //{name: "READ_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, - //{name: "WRITE_BANDWIDTH", tp: mysql.TypeLonglong, size: 21}, {name: "BURSTABLE", tp: mysql.TypeVarchar, size: 3}, } From c43f49e127e716639dcaf414c03c7dc7775237b5 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 3 Feb 2023 19:34:49 +0800 Subject: [PATCH 24/24] *: clean up code Signed-off-by: BornChanger --- executor/infoschema_reader.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 29d3896a2c72c..39666333762ab 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -3400,18 +3400,13 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { burstable := "NO" switch group.Mode { case rmpb.GroupMode_RUMode: - //mode = "RU_MODE" if group.RUSettings.RU.Settings.BurstLimit < 0 { burstable = "YES" } row := types.MakeDatums( group.Name, - //mode, group.RUSettings.RU.Settings.FillRate, uint64(group.RUSettings.RU.Tokens), - //nil, - //nil, - //nil, burstable, ) rows = append(rows, row) @@ -3419,10 +3414,6 @@ func (e *memtableRetriever) setDataFromResourceGroups() error { //mode = "UNKNOWN_MODE" row := types.MakeDatums( group.Name, - //mode, - //nil, - //nil, - //nil, nil, nil, nil,