Skip to content

Commit

Permalink
*: Support multi-column RANGE COLUMNS partitioning (#36637)
Browse files Browse the repository at this point in the history 
close #36636
  • Loading branch information
@mjonss
mjonss committed Sep 16, 2022
1 parent e2abe3d commit 8161947
Show file tree
Hide file tree
Showing 11 changed files with 1,369 additions and 245 deletions.
210 changes: 174 additions & 36 deletions ddl/db_partition_test.go
View file

Large diffs are not rendered by default.

27 changes: 17 additions & 10 deletions ddl/ddl_api.go
View file
Expand Up @@ -2777,6 +2777,7 @@ func checkColumnsPartitionType(tbInfo *model.TableInfo) error {
// DATE and DATETIME
// CHAR, VARCHAR, BINARY, and VARBINARY
// See https://dev.mysql.com/doc/mysql-partitioning-excerpt/5.7/en/partitioning-columns.html
// Note that also TIME is allowed in MySQL. Also see https://bugs.mysql.com/bug.php?id=84362
switch colInfo.FieldType.GetType() {
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeDuration:
Expand Down Expand Up @@ -6461,20 +6462,21 @@ func buildAddedPartitionDefs(ctx sessionctx.Context, meta *model.TableInfo, spec
return GeneratePartDefsFromInterval(ctx, spec.Tp, meta, spec.Partition)
}

func checkColumnsTypeAndValuesMatch(ctx sessionctx.Context, meta *model.TableInfo, exprs []ast.ExprNode) error {
func checkAndGetColumnsTypeAndValuesMatch(ctx sessionctx.Context, colTypes []types.FieldType, exprs []ast.ExprNode) ([]string, error) {
// Validate() has already checked len(colNames) = len(exprs)
// create table ... partition by range columns (cols)
// partition p0 values less than (expr)
// check the type of cols[i] and expr is consistent.
colTypes := collectColumnsType(meta)
valStrings := make([]string, 0, len(colTypes))
for i, colExpr := range exprs {
if _, ok := colExpr.(*ast.MaxValueExpr); ok {
valStrings = append(valStrings, partitionMaxValue)
continue
}
colType := colTypes[i]
val, err := expression.EvalAstExpr(ctx, colExpr)
if err != nil {
return err
return nil, err
}
// Check val.ConvertTo(colType) doesn't work, so we need this case by case check.
vkind := val.Kind()
Expand All @@ -6483,33 +6485,38 @@ func checkColumnsTypeAndValuesMatch(ctx sessionctx.Context, meta *model.TableInf
switch vkind {
case types.KindString, types.KindBytes:
default:
return dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
}
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
switch vkind {
case types.KindInt64, types.KindUint64, types.KindNull:
default:
return dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
}
case mysql.TypeFloat, mysql.TypeDouble:
switch vkind {
case types.KindFloat32, types.KindFloat64, types.KindNull:
default:
return dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
}
case mysql.TypeString, mysql.TypeVarString:
switch vkind {
case types.KindString, types.KindBytes, types.KindNull, types.KindBinaryLiteral:
default:
return dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
}
}
_, err = val.ConvertTo(ctx.GetSessionVars().StmtCtx, &colType)
newVal, err := val.ConvertTo(ctx.GetSessionVars().StmtCtx, &colType)
if err != nil {
return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
}
s, err := newVal.ToString()
if err != nil {
return dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
return nil, err
}
valStrings = append(valStrings, s)
}
return nil
return valStrings, nil
}

// LockTables uses to execute lock tables statement.
Expand Down
119 changes: 97 additions & 22 deletions ddl/partition.go
View file
Expand Up @@ -17,6 +17,7 @@ package ddl
import (
"bytes"
"context"
"encoding/hex"
"fmt"
"math"
"strconv"
Expand Down Expand Up @@ -420,14 +421,7 @@ func buildTablePartitionInfo(ctx sessionctx.Context, s *ast.PartitionOptions, tb
switch s.Tp {
case model.PartitionTypeRange:
if s.Sub == nil {
// Partition by range expression is enabled by default.
if s.ColumnNames == nil {
enable = true
}
// Partition by range columns and just one column.
if len(s.ColumnNames) == 1 {
enable = true
}
enable = true
}
case model.PartitionTypeHash:
// Partition by hash is enabled by default.
Expand Down Expand Up @@ -784,10 +778,11 @@ func generatePartitionDefinitionsFromInterval(ctx sessionctx.Context, partOption
Exprs: []ast.ExprNode{*partOptions.Interval.LastRangeEnd},
}
if len(tbInfo.Partition.Columns) > 0 {
if err := checkColumnsTypeAndValuesMatch(ctx, tbInfo, first.Exprs); err != nil {
colTypes := collectColumnsType(tbInfo)
if _, err := checkAndGetColumnsTypeAndValuesMatch(ctx, colTypes, first.Exprs); err != nil {
return err
}
if err := checkColumnsTypeAndValuesMatch(ctx, tbInfo, last.Exprs); err != nil {
if _, err := checkAndGetColumnsTypeAndValuesMatch(ctx, colTypes, last.Exprs); err != nil {
return err
}
} else {
Expand Down Expand Up @@ -1082,14 +1077,17 @@ func buildHashPartitionDefinitions(_ sessionctx.Context, defs []*ast.PartitionDe
func buildListPartitionDefinitions(ctx sessionctx.Context, defs []*ast.PartitionDefinition, tbInfo *model.TableInfo) ([]model.PartitionDefinition, error) {
definitions := make([]model.PartitionDefinition, 0, len(defs))
exprChecker := newPartitionExprChecker(ctx, nil, checkPartitionExprAllowed)
colTypes := collectColumnsType(tbInfo)
for _, def := range defs {
if err := def.Clause.Validate(model.PartitionTypeList, len(tbInfo.Partition.Columns)); err != nil {
return nil, err
}
clause := def.Clause.(*ast.PartitionDefinitionClauseIn)
if len(tbInfo.Partition.Columns) > 0 {
for _, vs := range clause.Values {
if err := checkColumnsTypeAndValuesMatch(ctx, tbInfo, vs); err != nil {
// TODO: use the generated strings / normalized partition values
_, err := checkAndGetColumnsTypeAndValuesMatch(ctx, colTypes, vs)
if err != nil {
return nil, err
}
}
Expand Down Expand Up @@ -1150,13 +1148,16 @@ func collectColumnsType(tbInfo *model.TableInfo) []types.FieldType {
func buildRangePartitionDefinitions(ctx sessionctx.Context, defs []*ast.PartitionDefinition, tbInfo *model.TableInfo) ([]model.PartitionDefinition, error) {
definitions := make([]model.PartitionDefinition, 0, len(defs))
exprChecker := newPartitionExprChecker(ctx, nil, checkPartitionExprAllowed)
colTypes := collectColumnsType(tbInfo)
for _, def := range defs {
if err := def.Clause.Validate(model.PartitionTypeRange, len(tbInfo.Partition.Columns)); err != nil {
return nil, err
}
clause := def.Clause.(*ast.PartitionDefinitionClauseLessThan)
var partValStrings []string
if len(tbInfo.Partition.Columns) > 0 {
if err := checkColumnsTypeAndValuesMatch(ctx, tbInfo, clause.Exprs); err != nil {
var err error
if partValStrings, err = checkAndGetColumnsTypeAndValuesMatch(ctx, colTypes, clause.Exprs); err != nil {
return nil, err
}
} else {
Expand Down Expand Up @@ -1184,14 +1185,31 @@ func buildRangePartitionDefinitions(ctx sessionctx.Context, defs []*ast.Partitio

buf := new(bytes.Buffer)
// Range columns partitions support multi-column partitions.
for _, expr := range clause.Exprs {
for i, expr := range clause.Exprs {
expr.Accept(exprChecker)
if exprChecker.err != nil {
return nil, exprChecker.err
}
expr.Format(buf)
piDef.LessThan = append(piDef.LessThan, buf.String())
buf.Reset()
// If multi-column use new evaluated+normalized output, instead of just formatted expression
if len(partValStrings) > i && len(colTypes) > 1 {
partVal := partValStrings[i]
switch colTypes[i].EvalType() {
case types.ETInt:
// no wrapping
case types.ETDatetime, types.ETString, types.ETDuration:
if _, ok := clause.Exprs[i].(*ast.MaxValueExpr); !ok {
// Don't wrap MAXVALUE
partVal = driver.WrapInSingleQuotes(partVal)
}
default:
return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
}
piDef.LessThan = append(piDef.LessThan, partVal)
} else {
expr.Format(buf)
piDef.LessThan = append(piDef.LessThan, buf.String())
buf.Reset()
}
}
definitions = append(definitions, piDef)
}
Expand Down Expand Up @@ -2170,7 +2188,7 @@ func checkExchangePartitionRecordValidation(w *worker, pt *model.TableInfo, inde
// For range expression and range columns
if len(pi.Columns) == 0 {
sql, paramList = buildCheckSQLForRangeExprPartition(pi, index, schemaName, tableName)
} else if len(pi.Columns) == 1 {
} else {
sql, paramList = buildCheckSQLForRangeColumnsPartition(pi, index, schemaName, tableName)
}
case model.PartitionTypeList:
Expand Down Expand Up @@ -2736,6 +2754,56 @@ func checkNoTimestampArgs(tbInfo *model.TableInfo, exprs ...ast.ExprNode) error
return nil
}

// hexIfNonPrint checks if printable UTF-8 characters from a single quoted string,
// if so, just returns the string
// else returns a hex string of the binary string (i.e. actual encoding, not unicode code points!)
func hexIfNonPrint(s string) string {
isPrint := true
// https://go.dev/blog/strings `for range` of string converts to runes!
for _, runeVal := range s {
if !strconv.IsPrint(runeVal) {
isPrint = false
break
}
}
if isPrint {
return s
}
// To avoid 'simple' MySQL accepted escape characters, to be showed as hex, just escape them
// \0 \b \n \r \t \Z, see https://dev.mysql.com/doc/refman/8.0/en/string-literals.html
isPrint = true
res := ""
for _, runeVal := range s {
switch runeVal {
case 0: // Null
res += `\0`
case 7: // Bell
res += `\b`
case '\t': // 9
res += `\t`
case '\n': // 10
res += `\n`
case '\r': // 13
res += `\r`
case 26: // ctrl-z / Substitute
res += `\Z`
default:
if strconv.IsPrint(runeVal) {
res += string(runeVal)
} else {
isPrint = false
break
}
}
}
if isPrint {
return res
}
// Not possible to create an easy interpreted MySQL string, return as hex string
// Can be converted to string in MySQL like: CAST(UNHEX('<hex string>') AS CHAR(255))
return "0x" + hex.EncodeToString([]byte(driver.UnwrapFromSingleQuotes(s)))
}

// AppendPartitionDefs generates a list of partition definitions needed for SHOW CREATE TABLE (in executor/show.go)
// as well as needed for generating the ADD PARTITION query for INTERVAL partitioning of ALTER TABLE t LAST PARTITION
// and generating the CREATE TABLE query from CREATE TABLE ... INTERVAL
Expand All @@ -2747,8 +2815,11 @@ func AppendPartitionDefs(partitionInfo *model.PartitionInfo, buf *bytes.Buffer,
fmt.Fprintf(buf, "PARTITION %s", stringutil.Escape(def.Name.O, sqlMode))
// PartitionTypeHash does not have any VALUES definition
if partitionInfo.Type == model.PartitionTypeRange {
lessThans := strings.Join(def.LessThan, ",")
fmt.Fprintf(buf, " VALUES LESS THAN (%s)", lessThans)
lessThans := make([]string, len(def.LessThan))
for idx, v := range def.LessThan {
lessThans[idx] = hexIfNonPrint(v)
}
fmt.Fprintf(buf, " VALUES LESS THAN (%s)", strings.Join(lessThans, ","))
} else if partitionInfo.Type == model.PartitionTypeList {
values := bytes.NewBuffer(nil)
for j, inValues := range def.InValues {
Expand All @@ -2757,10 +2828,14 @@ func AppendPartitionDefs(partitionInfo *model.PartitionInfo, buf *bytes.Buffer,
}
if len(inValues) > 1 {
values.WriteString("(")
values.WriteString(strings.Join(inValues, ","))
tmpVals := make([]string, len(inValues))
for idx, v := range inValues {
tmpVals[idx] = hexIfNonPrint(v)
}
values.WriteString(strings.Join(tmpVals, ","))
values.WriteString(")")
} else {
values.WriteString(strings.Join(inValues, ","))
} else if len(inValues) == 1 {
values.WriteString(hexIfNonPrint(inValues[0]))
}
}
fmt.Fprintf(buf, " VALUES IN (%s)", values.String())
Expand Down
13 changes: 9 additions & 4 deletions executor/seqtest/seq_executor_test.go
View file
Expand Up @@ -504,19 +504,24 @@ func TestShow(t *testing.T) {

// Test range columns partition
tk.MustExec(`drop table if exists t`)
tk.MustExec(`CREATE TABLE t (a int, b int, c char, d int) PARTITION BY RANGE COLUMNS(a,d,c) (
tk.MustExec(`CREATE TABLE t (a int, b int, c varchar(25), d int) PARTITION BY RANGE COLUMNS(a,d,c) (
PARTITION p0 VALUES LESS THAN (5,10,'ggg'),
PARTITION p1 VALUES LESS THAN (10,20,'mmm'),
PARTITION p2 VALUES LESS THAN (15,30,'sss'),
PARTITION p3 VALUES LESS THAN (50,MAXVALUE,MAXVALUE))`)
tk.MustQuery("show warnings").Check(testkit.Rows())
tk.MustQuery("show create table t").Check(testkit.RowsWithSep("|",
"t CREATE TABLE `t` (\n"+
" `a` int(11) DEFAULT NULL,\n"+
" `b` int(11) DEFAULT NULL,\n"+
" `c` char(1) DEFAULT NULL,\n"+
" `c` varchar(25) DEFAULT NULL,\n"+
" `d` int(11) DEFAULT NULL\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin",
))
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n"+
"PARTITION BY RANGE COLUMNS(`a`,`d`,`c`)\n"+
"(PARTITION `p0` VALUES LESS THAN (5,10,'ggg'),\n"+
" PARTITION `p1` VALUES LESS THAN (10,20,'mmm'),\n"+
" PARTITION `p2` VALUES LESS THAN (15,30,'sss'),\n"+
" PARTITION `p3` VALUES LESS THAN (50,MAXVALUE,MAXVALUE))"))

// Test hash partition
tk.MustExec(`drop table if exists t`)
Expand Down
37 changes: 27 additions & 10 deletions executor/showtest/show_test.go
View file
Expand Up @@ -472,6 +472,17 @@ func TestShowCreateTable(t *testing.T) {
"t CREATE TABLE `t` (\n"+
" `a` bit(1) DEFAULT rand()\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"))

tk.MustExec(`drop table if exists t`)
err := tk.ExecToErr(`create table t (a varchar(255) character set ascii) partition by range columns (a) (partition p values less than (0xff))`)
require.ErrorContains(t, err, "[ddl:1654]Partition column values of incorrect type")
tk.MustExec(`create table t (a varchar(255) character set ascii) partition by range columns (a) (partition p values less than (0x7f))`)
tk.MustQuery(`show create table t`).Check(testkit.Rows(
"t CREATE TABLE `t` (\n" +
" `a` varchar(255) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\n" +
"PARTITION BY RANGE COLUMNS(`a`)\n" +
"(PARTITION `p` VALUES LESS THAN (x'7f'))"))
}

func TestShowCreateTablePlacement(t *testing.T) {
Expand Down Expand Up @@ -589,21 +600,27 @@ func TestShowCreateTablePlacement(t *testing.T) {
))

tk.MustExec(`DROP TABLE IF EXISTS t`)
// RANGE COLUMNS with multiple columns is not supported!

tk.MustExec("create table t(a int, b varchar(255))" +
"/*T![placement] PLACEMENT POLICY=\"x\" */" +
"PARTITION BY RANGE COLUMNS (a,b)\n" +
"(PARTITION pLow VALUES less than (1000000,'1000000') COMMENT 'a comment' placement policy 'x'," +
" PARTITION pMidLow VALUES less than (1000000,MAXVALUE) COMMENT 'another comment' placement policy 'x'," +
" PARTITION pMadMax VALUES less than (MAXVALUE,'1000000') COMMENT ='Not a comment' placement policy 'x'," +
"partition pMax values LESS THAN (MAXVALUE, MAXVALUE))")
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 8200 Unsupported partition type RANGE, treat as normal table"))
tk.MustQuery(`show create table t`).Check(testkit.RowsWithSep("|", ""+
"t CREATE TABLE `t` (\n"+
" `a` int(11) DEFAULT NULL,\n"+
" `b` varchar(255) DEFAULT NULL\n"+
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![placement] PLACEMENT POLICY=`x` */",
))
" PARTITION pMadMax VALUES less than (9000000,'1000000') COMMENT ='Not a comment' placement policy 'x'," +
"partition pMax values LESS THAN (MAXVALUE, 'Does not matter...'))")
tk.MustQuery("show warnings").Check(testkit.Rows())
tk.MustExec(`insert into t values (1,'1')`)
tk.MustQuery("select * from t").Check(testkit.Rows("1 1"))
tk.MustQuery(`show create table t`).Check(testkit.Rows(
"t CREATE TABLE `t` (\n" +
" `a` int(11) DEFAULT NULL,\n" +
" `b` varchar(255) DEFAULT NULL\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T![placement] PLACEMENT POLICY=`x` */\n" +
"PARTITION BY RANGE COLUMNS(`a`,`b`)\n" +
"(PARTITION `pLow` VALUES LESS THAN (1000000,'1000000') COMMENT 'a comment' /*T![placement] PLACEMENT POLICY=`x` */,\n" +
" PARTITION `pMidLow` VALUES LESS THAN (1000000,MAXVALUE) COMMENT 'another comment' /*T![placement] PLACEMENT POLICY=`x` */,\n" +
" PARTITION `pMadMax` VALUES LESS THAN (9000000,'1000000') COMMENT 'Not a comment' /*T![placement] PLACEMENT POLICY=`x` */,\n" +
" PARTITION `pMax` VALUES LESS THAN (MAXVALUE,'Does not matter...'))"))

tk.MustExec(`DROP TABLE IF EXISTS t`)
tk.MustExec("create table t(a int, b varchar(255))" +
Expand Down

0 comments on commit 8161947

Please sign in to comment.