Merge pull request #7444 from planetscale/table-size

Publish table size on schema

go/mysql/fakesqldb/server.go

@@ -136,6 +136,7 @@ type ExpectedResult struct {
 type exprResult struct {
 	expr   *regexp.Regexp
 	result *sqltypes.Result
+	err    string
 }
 
 // ExpectedExecuteFetch defines for an expected query the to be faked output.
@@ -391,6 +392,9 @@ func (db *DB) HandleQuery(c *mysql.Conn, query string, callback func(*sqltypes.R
 			if ok {
 				userCallback(query)
 			}
+			if pat.err != "" {
+				return fmt.Errorf(pat.err)
+			}
 			return callback(pat.result)
 		}
 	}
@@ -504,7 +508,20 @@ func (db *DB) AddQueryPattern(queryPattern string, expectedResult *sqltypes.Resu
 	result := *expectedResult
 	db.mu.Lock()
 	defer db.mu.Unlock()
-	db.patternData = append(db.patternData, exprResult{expr, &result})
+	db.patternData = append(db.patternData, exprResult{expr: expr, result: &result})
+}
+
+// RejectQueryPattern allows a query pattern to be rejected with an error
+func (db *DB) RejectQueryPattern(queryPattern, error string) {
+	expr := regexp.MustCompile("(?is)^" + queryPattern + "$")
+	db.mu.Lock()
+	defer db.mu.Unlock()
+	db.patternData = append(db.patternData, exprResult{expr: expr, err: error})
+}
+
+// ClearQueryPattern removes all query patterns set up
+func (db *DB) ClearQueryPattern() {
+	db.patternData = nil
 }
 
 // AddQueryPatternWithCallback is similar to AddQueryPattern: in addition it calls the provided callback function

go/mysql/flavor.go

@@ -44,6 +44,10 @@ const (
 	mariaDBReplicationHackPrefix = "5.5.5-"
 	// mariaDBVersionString is present in
 	mariaDBVersionString = "MariaDB"
+	// mysql57VersionPrefix is the prefix for 5.7 mysql version, such as 5.7.31-log
+	mysql57VersionPrefix = "5.7."
+	// mysql80VersionPrefix is the prefix for 8.0 mysql version, such as 8.0.19
+	mysql80VersionPrefix = "8.0."
 )
 
 // flavor is the abstract interface for a flavor.
@@ -111,6 +115,8 @@ type flavor interface {
 	// timestamp cannot be set by regular clients.
 	enableBinlogPlaybackCommand() string
 	disableBinlogPlaybackCommand() string
+
+	baseShowTablesWithSizes() string
 }
 
 // flavors maps flavor names to their implementation.
@@ -131,23 +137,27 @@ var flavors = make(map[string]func() flavor)
 // as well (not matching what c.ServerVersion is, but matching after we remove
 // the prefix).
 func (c *Conn) fillFlavor(params *ConnParams) {
-	if flavorFunc := flavors[params.Flavor]; flavorFunc != nil {
-		c.flavor = flavorFunc()
-		return
-	}
+	flavorFunc := flavors[params.Flavor]
 
-	if strings.HasPrefix(c.ServerVersion, mariaDBReplicationHackPrefix) {
+	switch {
+	case flavorFunc != nil:
+		c.flavor = flavorFunc()
+	case strings.HasPrefix(c.ServerVersion, mariaDBReplicationHackPrefix):
 		c.ServerVersion = c.ServerVersion[len(mariaDBReplicationHackPrefix):]
-		c.flavor = mariadbFlavor{}
-		return
-	}
-
-	if strings.Contains(c.ServerVersion, mariaDBVersionString) {
-		c.flavor = mariadbFlavor{}
-		return
+		c.flavor = mariadbFlavor101{}
+	case strings.Contains(c.ServerVersion, mariaDBVersionString):
+		mariadbVersion, err := strconv.ParseFloat(c.ServerVersion[:4], 64)
+		if err != nil || mariadbVersion < 10.2 {
+			c.flavor = mariadbFlavor101{}
+		}
+		c.flavor = mariadbFlavor102{}
+	case strings.HasPrefix(c.ServerVersion, mysql57VersionPrefix):
+		c.flavor = mysqlFlavor57{}
+	case strings.HasPrefix(c.ServerVersion, mysql80VersionPrefix):
+		c.flavor = mysqlFlavor80{}
+	default:
+		c.flavor = mysqlFlavor56{}
 	}
-
-	c.flavor = mysqlFlavor{}
 }
 
 //
@@ -159,8 +169,11 @@ func (c *Conn) fillFlavor(params *ConnParams) {
 // is identified as MariaDB. Most applications should not care, but
 // this is useful in tests.
 func (c *Conn) IsMariaDB() bool {
-	_, ok := c.flavor.(mariadbFlavor)
-	return ok
+	switch c.flavor.(type) {
+	case mariadbFlavor101, mariadbFlavor102:
+		return true
+	}
+	return false
 }
 
 // MasterPosition returns the current master replication position.
@@ -390,3 +403,8 @@ func (c *Conn) EnableBinlogPlaybackCommand() string {
 func (c *Conn) DisableBinlogPlaybackCommand() string {
 	return c.flavor.disableBinlogPlaybackCommand()
 }
+
+// BaseShowTables returns a query that shows tables and their sizes
+func (c *Conn) BaseShowTables() string {
+	return c.flavor.baseShowTablesWithSizes()
+}

go/mysql/flavor_filepos.go

@@ -271,3 +271,8 @@ func (*filePosFlavor) enableBinlogPlaybackCommand() string {
 func (*filePosFlavor) disableBinlogPlaybackCommand() string {
 	return ""
 }
+
+// baseShowTablesWithSizes is part of the Flavor interface.
+func (*filePosFlavor) baseShowTablesWithSizes() string {
+	return TablesWithSize56
+}

go/mysql/flavor_mariadb.go

@@ -30,6 +30,15 @@ import (
 
 // mariadbFlavor implements the Flavor interface for MariaDB.
 type mariadbFlavor struct{}
+type mariadbFlavor101 struct {
+	mariadbFlavor
+}
+type mariadbFlavor102 struct {
+	mariadbFlavor
+}
+
+var _ flavor = (*mariadbFlavor101)(nil)
+var _ flavor = (*mariadbFlavor102)(nil)
 
 // masterGTIDSet is part of the Flavor interface.
 func (mariadbFlavor) masterGTIDSet(c *Conn) (GTIDSet, error) {

go/mysql/flavor_mariadb_binlog_playback.go

@@ -29,3 +29,13 @@ func (mariadbFlavor) enableBinlogPlaybackCommand() string {
 func (mariadbFlavor) disableBinlogPlaybackCommand() string {
 	return ""
 }
+
+// baseShowTablesWithSizes is part of the Flavor interface.
+func (mariadbFlavor101) baseShowTablesWithSizes() string {
+	return TablesWithSize56
+}
+
+// baseShowTablesWithSizes is part of the Flavor interface.
+func (mariadbFlavor102) baseShowTablesWithSizes() string {
+	return TablesWithSize57
+}

go/mysql/flavor_mariadb_test.go

@@ -40,7 +40,7 @@ func TestMariadbSetMasterCommands(t *testing.T) {
   MASTER_CONNECT_RETRY = 1234,
   MASTER_USE_GTID = current_pos`
 
-	conn := &Conn{flavor: mariadbFlavor{}}
+	conn := &Conn{flavor: mariadbFlavor101{}}
 	got := conn.SetMasterCommand(params, masterHost, masterPort, masterConnectRetry)
 	if got != want {
 		t.Errorf("mariadbFlavor.SetMasterCommands(%#v, %#v, %#v, %#v) = %#v, want %#v", params, masterHost, masterPort, masterConnectRetry, got, want)
@@ -73,7 +73,7 @@ func TestMariadbSetMasterCommandsSSL(t *testing.T) {
   MASTER_SSL_KEY = 'ssl-key',
   MASTER_USE_GTID = current_pos`
 
-	conn := &Conn{flavor: mariadbFlavor{}}
+	conn := &Conn{flavor: mariadbFlavor101{}}
 	got := conn.SetMasterCommand(params, masterHost, masterPort, masterConnectRetry)
 	if got != want {
 		t.Errorf("mariadbFlavor.SetMasterCommands(%#v, %#v, %#v, %#v) = %#v, want %#v", params, masterHost, masterPort, masterConnectRetry, got, want)

go/mysql/flavor_mysql.go

@@ -29,6 +29,19 @@ import (
 
 // mysqlFlavor implements the Flavor interface for Mysql.
 type mysqlFlavor struct{}
+type mysqlFlavor56 struct {
+	mysqlFlavor
+}
+type mysqlFlavor57 struct {
+	mysqlFlavor
+}
+type mysqlFlavor80 struct {
+	mysqlFlavor
+}
+
+var _ flavor = (*mysqlFlavor56)(nil)
+var _ flavor = (*mysqlFlavor57)(nil)
+var _ flavor = (*mysqlFlavor80)(nil)
 
 // masterGTIDSet is part of the Flavor interface.
 func (mysqlFlavor) masterGTIDSet(c *Conn) (GTIDSet, error) {
@@ -231,3 +244,40 @@ func (mysqlFlavor) enableBinlogPlaybackCommand() string {
 func (mysqlFlavor) disableBinlogPlaybackCommand() string {
 	return ""
 }
+
+// TablesWithSize56 is a query to select table along with size for mysql 5.6
+const TablesWithSize56 = `SELECT table_name, table_type, unix_timestamp(create_time), table_comment, SUM( data_length + index_length), SUM( data_length + index_length) 
+		FROM information_schema.tables WHERE table_schema = database() group by table_name`
+
+// TablesWithSize57 is a query to select table along with size for mysql 5.7.
+// It's a little weird, because the JOIN predicate only works if the table and databases do not contain weird characters.
+// As a fallback, we use the mysql 5.6 query, which is not always up to date, but works for all table/db names.
+const TablesWithSize57 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size 
+	FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i 
+	WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)
+UNION ALL
+	SELECT table_name, table_type, unix_timestamp(create_time), table_comment, SUM( data_length + index_length), SUM( data_length + index_length)
+	FROM information_schema.tables t
+	WHERE table_schema = database() AND NOT EXISTS(SELECT * FROM information_schema.innodb_sys_tablespaces i WHERE i.name = concat(t.table_schema,'/',t.table_name)) 
+	group by table_name, table_type, unix_timestamp(create_time), table_comment
+`
+
+// TablesWithSize80 is a query to select table along with size for mysql 8.0
+const TablesWithSize80 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size 
+		FROM information_schema.tables t, information_schema.innodb_tablespaces i 
+		WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)`
+
+// baseShowTablesWithSizes is part of the Flavor interface.
+func (mysqlFlavor56) baseShowTablesWithSizes() string {
+	return TablesWithSize56
+}
+
+// baseShowTablesWithSizes is part of the Flavor interface.
+func (mysqlFlavor57) baseShowTablesWithSizes() string {
+	return TablesWithSize57
+}
+
+// baseShowTablesWithSizes is part of the Flavor interface.
+func (mysqlFlavor80) baseShowTablesWithSizes() string {
+	return TablesWithSize80
+}

go/mysql/flavor_mysql_test.go

@@ -39,7 +39,7 @@ func TestMysql56SetMasterCommands(t *testing.T) {
   MASTER_CONNECT_RETRY = 1234,
   MASTER_AUTO_POSITION = 1`
 
-	conn := &Conn{flavor: mysqlFlavor{}}
+	conn := &Conn{flavor: mysqlFlavor57{}}
 	got := conn.SetMasterCommand(params, masterHost, masterPort, masterConnectRetry)
 	if got != want {
 		t.Errorf("mysqlFlavor.SetMasterCommand(%#v, %#v, %#v, %#v) = %#v, want %#v", params, masterHost, masterPort, masterConnectRetry, got, want)
@@ -72,7 +72,7 @@ func TestMysql56SetMasterCommandsSSL(t *testing.T) {
   MASTER_SSL_KEY = 'ssl-key',
   MASTER_AUTO_POSITION = 1`
 
-	conn := &Conn{flavor: mysqlFlavor{}}
+	conn := &Conn{flavor: mysqlFlavor57{}}
 	got := conn.SetMasterCommand(params, masterHost, masterPort, masterConnectRetry)
 	if got != want {
 		t.Errorf("mysqlFlavor.SetMasterCommands(%#v, %#v, %#v, %#v) = %#v, want %#v", params, masterHost, masterPort, masterConnectRetry, got, want)

go/mysql/schema.go

@@ -30,58 +30,62 @@ import (
 // data.
 
 const (
-	// BaseShowTables is the base query used in further methods.
-	BaseShowTables = "SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()"
-
 	// BaseShowPrimary is the base query for fetching primary key info.
 	BaseShowPrimary = "SELECT table_name, column_name FROM information_schema.key_column_usage WHERE table_schema=database() AND constraint_name='PRIMARY' ORDER BY table_name, ordinal_position"
 )
 
 // BaseShowTablesFields contains the fields returned by a BaseShowTables or a BaseShowTablesForTable command.
 // They are validated by the
 // testBaseShowTables test.
-var BaseShowTablesFields = []*querypb.Field{
-	{
-		Name:         "table_name",
-		Type:         querypb.Type_VARCHAR,
-		Table:        "tables",
-		OrgTable:     "TABLES",
-		Database:     "information_schema",
-		OrgName:      "TABLE_NAME",
-		ColumnLength: 192,
-		Charset:      CharacterSetUtf8,
-		Flags:        uint32(querypb.MySqlFlag_NOT_NULL_FLAG),
-	},
-	{
-		Name:         "table_type",
-		Type:         querypb.Type_VARCHAR,
-		Table:        "tables",
-		OrgTable:     "TABLES",
-		Database:     "information_schema",
-		OrgName:      "TABLE_TYPE",
-		ColumnLength: 192,
-		Charset:      CharacterSetUtf8,
-		Flags:        uint32(querypb.MySqlFlag_NOT_NULL_FLAG),
-	},
-	{
-		Name:         "unix_timestamp(create_time)",
-		Type:         querypb.Type_INT64,
-		ColumnLength: 11,
-		Charset:      CharacterSetBinary,
-		Flags:        uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG),
-	},
-	{
-		Name:         "table_comment",
-		Type:         querypb.Type_VARCHAR,
-		Table:        "tables",
-		OrgTable:     "TABLES",
-		Database:     "information_schema",
-		OrgName:      "TABLE_COMMENT",
-		ColumnLength: 6144,
-		Charset:      CharacterSetUtf8,
-		Flags:        uint32(querypb.MySqlFlag_NOT_NULL_FLAG),
-	},
-}
+var BaseShowTablesFields = []*querypb.Field{{
+	Name:         "t.table_name",
+	Type:         querypb.Type_VARCHAR,
+	Table:        "tables",
+	OrgTable:     "TABLES",
+	Database:     "information_schema",
+	OrgName:      "TABLE_NAME",
+	ColumnLength: 192,
+	Charset:      CharacterSetUtf8,
+	Flags:        uint32(querypb.MySqlFlag_NOT_NULL_FLAG),
+}, {
+	Name:         "t.table_type",
+	Type:         querypb.Type_VARCHAR,
+	Table:        "tables",
+	OrgTable:     "TABLES",
+	Database:     "information_schema",
+	OrgName:      "TABLE_TYPE",
+	ColumnLength: 192,
+	Charset:      CharacterSetUtf8,
+	Flags:        uint32(querypb.MySqlFlag_NOT_NULL_FLAG),
+}, {
+	Name:         "unix_timestamp(t.create_time)",
+	Type:         querypb.Type_INT64,
+	ColumnLength: 11,
+	Charset:      CharacterSetBinary,
+	Flags:        uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG),
+}, {
+	Name:         "t.table_comment",
+	Type:         querypb.Type_VARCHAR,
+	Table:        "tables",
+	OrgTable:     "TABLES",
+	Database:     "information_schema",
+	OrgName:      "TABLE_COMMENT",
+	ColumnLength: 6144,
+	Charset:      CharacterSetUtf8,
+	Flags:        uint32(querypb.MySqlFlag_NOT_NULL_FLAG),
+}, {
+	Name:         "i.file_size",
+	Type:         querypb.Type_INT64,
+	ColumnLength: 11,
+	Charset:      CharacterSetBinary,
+	Flags:        uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG),
+}, {
+	Name:         "i.allocated_size",
+	Type:         querypb.Type_INT64,
+	ColumnLength: 11,
+	Charset:      CharacterSetBinary,
+	Flags:        uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG),
+}}
 
 // BaseShowTablesRow returns the fields from a BaseShowTables or
 // BaseShowTablesForTable command.
@@ -95,6 +99,8 @@ func BaseShowTablesRow(tableName string, isView bool, comment string) []sqltypes
 		sqltypes.MakeTrusted(sqltypes.VarChar, []byte(tableType)),
 		sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325875")), // unix_timestamp(create_time)
 		sqltypes.MakeTrusted(sqltypes.VarChar, []byte(comment)),
+		sqltypes.MakeTrusted(sqltypes.Int64, []byte("100")), // file_size
+		sqltypes.MakeTrusted(sqltypes.Int64, []byte("150")), // allocated_size
 	}
 }
 

go/vt/vtexplain/vtexplain_vttablet.go

@@ -382,7 +382,7 @@ func initTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options) error {
 		}
 		showTableRows = append(showTableRows, mysql.BaseShowTablesRow(table, false, options))
 	}
-	schemaQueries[mysql.BaseShowTables] = &sqltypes.Result{
+	schemaQueries[mysql.TablesWithSize57] = &sqltypes.Result{
 		Fields: mysql.BaseShowTablesFields,
 		Rows:   showTableRows,
 	}

go/vt/vttablet/tabletserver/connpool/dbconn.go

@@ -374,6 +374,11 @@ func (dbc *DBConn) ID() int64 {
 	return dbc.conn.ID()
 }
 
+// BaseShowTables returns a query that shows tables and their sizes
+func (dbc *DBConn) BaseShowTables() string {
+	return dbc.conn.BaseShowTables()
+}
+
 func (dbc *DBConn) reconnect(ctx context.Context) error {
 	dbc.conn.Close()
 	// Reuse MySQLTimings from dbc.conn.