Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: add split index region syntax #10203

Merged
merged 30 commits into from May 6, 2019
Merged
Changes from 25 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
5c613ad
init
crazycs520 Apr 10, 2019
2bc50c8
add scatter
crazycs520 Apr 12, 2019
a88ea9b
fix test
crazycs520 Apr 12, 2019
3145795
only split region for positive num
crazycs520 Apr 12, 2019
2fccfab
add PreSplitRegions option
crazycs520 Apr 12, 2019
77f0a96
add retry for scatter region
crazycs520 Apr 12, 2019
189335d
refine comment
crazycs520 Apr 12, 2019
03783c0
Merge branch 'master' into splite-region
crazycs520 Apr 12, 2019
b517f03
remove blank line
crazycs520 Apr 15, 2019
f96d8d3
Merge branch 'master' into splite-region
crazycs520 Apr 16, 2019
55cd7be
update go.mod for pd
crazycs520 Apr 16, 2019
eb27dfb
add wait scatter finish
crazycs520 Apr 17, 2019
a70696f
add table option: wait_split_finish
crazycs520 Apr 17, 2019
df45ac4
Merge branch 'master' of https://github.com/pingcap/tidb into splite-…
crazycs520 Apr 18, 2019
e362d68
split and scatter region first, wait scatter finish together
crazycs520 Apr 18, 2019
918b183
*: add split table index syntax
crazycs520 Apr 19, 2019
87ec41a
wait scatter finish when create table
crazycs520 Apr 22, 2019
58c4a9e
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 Apr 25, 2019
6cfd51e
refine code
crazycs520 Apr 25, 2019
f1b22a3
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 Apr 25, 2019
952b870
add comment
crazycs520 Apr 25, 2019
38a057d
address comment and add test for convert value should not ignore trun…
crazycs520 Apr 26, 2019
53ccd4f
Merge branch 'master' into split_index
crazycs520 Apr 26, 2019
b44db0b
use session variable tidb_wait_table_split_finish to chose sync or as…
crazycs520 Apr 28, 2019
15c9514
update go mod
crazycs520 Apr 29, 2019
650c428
address comment
crazycs520 May 6, 2019
77ca0d1
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 May 6, 2019
1d2e9c9
address comment
crazycs520 May 6, 2019
44016a5
address comment and update go.mod
crazycs520 May 6, 2019
652e95c
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 May 6, 2019
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -194,6 +194,8 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor {
return b.buildWindow(v)
case *plannercore.SQLBindPlan:
return b.buildSQLBindExec(v)
case *plannercore.SplitIndexRegion:
return b.buildSplitIndexRegion(v)
default:
if mp, ok := p.(MockPhysicalPlan); ok {
return mp.GetExecutor()
@@ -1249,6 +1251,17 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo
return e
}

func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitIndexRegion) Executor {
base := newBaseExecutor(b.ctx, nil, v.ExplainID())
base.initCap = chunk.ZeroCapacity
return &SplitIndexRegionExec{
baseExecutor: base,
table: v.Table,
indexInfo: v.IndexInfo,
valueLists: v.ValueLists,
}
}

func (b *executorBuilder) buildUpdate(v *plannercore.Update) Executor {
tblID2table := make(map[int64]table.Table)
for id := range v.SelectPlan.Schema().TblID2Handle {
@@ -1379,6 +1379,10 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) {
sc.InShowWarning = true
sc.SetWarnings(vars.StmtCtx.GetWarnings())
}
case *ast.SplitIndexRegionStmt:
sc.IgnoreTruncate = false
sc.IgnoreZeroInDate = true
sc.AllowInvalidDate = vars.SQLMode.HasAllowInvalidDatesMode()
This conversation was marked as resolved by crazycs520

This comment has been minimized.

Copy link
@winkyao

winkyao May 6, 2019

Member

What is this field used for?

This comment has been minimized.

Copy link
@crazycs520

crazycs520 May 6, 2019

Author Contributor

Field sc.AllowInvalidDate just keep consistent with the default, I just want to set IgnoreTruncate to false here.

default:
sc.IgnoreTruncate = true
sc.IgnoreZeroInDate = true
@@ -3655,6 +3655,19 @@ func (s *testSuite) TestReadPartitionedTable(c *C) {
tk.MustQuery("select a from pt where b = 3").Check(testkit.Rows("3"))
}

func (s *testSuite) TestSplitIndexRegion(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a varchar(100),b int, index idx1(b,a))")
tk.MustExec(`split table t index idx1 by (10000,"abcd"),(10000000);`)
_, err := tk.Exec(`split table t index idx1 by ("abcd");`)
c.Assert(err, NotNil)
terr := errors.Cause(err).(*terror.Error)
c.Assert(terr.Code(), Equals, terror.ErrCode(mysql.WarnDataTruncated))

}

type testOOMSuite struct {
store kv.Storage
do *domain.Domain
@@ -0,0 +1,81 @@
// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package executor

import (
"context"

"github.com/pingcap/parser/model"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/logutil"
"go.uber.org/zap"
)

// SplitIndexRegionExec represents a split index regions executor.
type SplitIndexRegionExec struct {
baseExecutor

table table.Table
indexInfo *model.IndexInfo
valueLists [][]types.Datum
}

type splitableStore interface {
SplitRegionAndScatter(splitKey kv.Key) (uint64, error)
WaitScatterRegionFinish(regionID uint64) error
}

// Next implements the Executor Next interface.
func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) error {
store := e.ctx.GetStore()
s, ok := store.(splitableStore)
if !ok {
This conversation was marked as resolved by zimulala

This comment has been minimized.

Copy link
@zimulala

zimulala May 6, 2019

Member

Do we need to return an error?

This comment has been minimized.

Copy link
@crazycs520

crazycs520 May 6, 2019

Author Contributor

You mean when where is only 1 tidb-server and no tikv-server, then return error here?

This comment has been minimized.

Copy link
@crazycs520

This comment has been minimized.

Copy link
@zimulala

zimulala May 6, 2019

Member

Nope, I forgot it, sorry.

return nil
}
regionIDs := make([]uint64, 0, len(e.valueLists))
index := tables.NewIndex(e.table.Meta().ID, e.table.Meta(), e.indexInfo)
for _, values := range e.valueLists {
idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, values, 1, nil)

This comment has been minimized.

Copy link
@nolouch

nolouch Apr 25, 2019

Member

1 means min handle id?

This comment has been minimized.

Copy link
@crazycs520

crazycs520 Apr 26, 2019

Author Contributor

yes, Maybe 0 is better?

This comment has been minimized.

Copy link
@winkyao

winkyao Apr 29, 2019

Member

you can use a constant, better than hard code!

This comment has been minimized.

Copy link
@zimulala

zimulala Apr 29, 2019

Member

@winkyao
I think 1 isn't always the min handle. If a table has a PKIsHandle column, then the handle may be a negative value.
So I think we need to use math.MinInt64.

This comment has been minimized.

Copy link
@winkyao

winkyao May 6, 2019

Member

@zimulala The key is encoding in binary format, so negative value is always greater than a positive value. @nolouch Right?

This comment has been minimized.

Copy link
@crazycs520

crazycs520 May 6, 2019

Author Contributor

Here is a test:
0 encode to byte is: [128 0 0 0 0 0 0 0]
math.MinInt64 encode to byte is: [0 0 0 0 0 0 0 0]

so, math.MinInt64 is better.

if err != nil {
return err
}

regionID, err := s.SplitRegionAndScatter(idxKey)
if err != nil {
logutil.Logger(context.Background()).Warn("split table index region failed",
zap.String("table", e.table.Meta().Name.L),
zap.String("index", e.indexInfo.Name.L),
zap.Error(err))
continue
}
regionIDs = append(regionIDs, regionID)

}
if !e.ctx.GetSessionVars().WaitTableSplitFinish {
return nil
}
for _, regionID := range regionIDs {
err := s.WaitScatterRegionFinish(regionID)
if err != nil {
logutil.Logger(context.Background()).Warn("wait scatter region failed",
zap.Uint64("regionID", regionID),

This comment has been minimized.

Copy link
@zimulala

zimulala May 6, 2019

Member

Do we need to add the table and index information to this log?

This comment has been minimized.

Copy link
@crazycs520

crazycs520 May 6, 2019

Author Contributor

done.

zap.Error(err))
}
}
return nil
}
2 go.mod
@@ -72,3 +72,5 @@ require (
sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4
sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
)

replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190429011413-f0b2f675e283
4 go.sum
@@ -30,6 +30,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/crazycs520/parser v0.0.0-20190429011413-f0b2f675e283 h1:m1tZP0PGLZ6NIuApN7HG5gqv0t+ie+HWG0wNMUXJ4m0=
github.com/crazycs520/parser v0.0.0-20190429011413-f0b2f675e283/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE=
@@ -160,8 +162,6 @@ github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXC
github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w=
github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ=
github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw=
github.com/pingcap/parser v0.0.0-20190424024541-e2cdb851bce2 h1:WZPcqMEpW1PFsusiCEiNlatZdI8dGURXneIUrFNmehI=
github.com/pingcap/parser v0.0.0-20190424024541-e2cdb851bce2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 h1:ZoKjndm/Ig7Ru/wojrQkc/YLUttUdQXoH77gtuWCvL4=
github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM=
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU=
@@ -29,6 +29,7 @@ import (
"github.com/pingcap/tidb/metrics"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/types"
driver "github.com/pingcap/tidb/types/parser_driver"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/kvcache"
@@ -475,6 +476,15 @@ type LoadStats struct {
Path string
}

// SplitIndexRegion represents a split index regions plan.
type SplitIndexRegion struct {
baseSchemaProducer

Table table.Table
IndexInfo *model.IndexInfo
ValueLists [][]types.Datum
}

// DDL represents a DDL statement plan.
type DDL struct {
baseSchemaProducer
@@ -279,6 +279,8 @@ func (b *PlanBuilder) Build(node ast.Node) (Plan, error) {
return b.buildDropBindPlan(x)
case *ast.ChangeStmt:
return b.buildChange(x)
case *ast.SplitIndexRegionStmt:
return b.buildSplitIndexRegion(x)
}
return nil, ErrUnsupportedType.GenWithStack("Unsupported type %T", node)
}
@@ -1588,6 +1590,48 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan {
return p
}

func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Plan, error) {
tblInfo := node.Table.TableInfo
indexInfo := tblInfo.FindIndexByName(strings.ToLower(node.IndexName))
if indexInfo == nil {
return nil, ErrKeyDoesNotExist.GenWithStackByArgs(node.IndexName, tblInfo.Name)
}
schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo)
mockTablePlan := LogicalTableDual{}.Init(b.ctx)
This conversation was marked as resolved by crazycs520

This comment has been minimized.

Copy link
@winkyao

winkyao May 6, 2019

Member

What is mockTablePlan used for?

This comment has been minimized.

Copy link
@crazycs520

crazycs520 May 6, 2019

Author Contributor

It is useless now. Already delete this.

mockTablePlan.SetSchema(schema)

indexValues := make([][]types.Datum, 0, len(node.ValueLists))
for i, valuesItem := range node.ValueLists {
if len(valuesItem) > len(indexInfo.Columns) {
return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1)
}
valueList := make([]types.Datum, 0, len(valuesItem))
for j, valueItem := range valuesItem {
x, ok := valueItem.(*driver.ValueExpr)
if !ok {
return nil, errors.Trace(errors.New("expect constant values"))

This comment has been minimized.

Copy link
@zimulala

zimulala Apr 29, 2019

Member

Could we remove this trace? Other places don't use it.
And we can print the value to the error message.

This comment has been minimized.

Copy link
@crazycs520

crazycs520 May 6, 2019

Author Contributor

done.

}
value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[indexInfo.Columns[j].Offset].FieldType)
This conversation was marked as resolved by crazycs520

This comment has been minimized.

Copy link
@winkyao

winkyao May 6, 2019

Member
colOffset := indexInfo.Columns[j].Offset
value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType)

This comment has been minimized.

Copy link
@crazycs520

crazycs520 May 6, 2019

Author Contributor

done

if err != nil {
return nil, err
}

valueList = append(valueList, value)
}
indexValues = append(indexValues, valueList)
}
tableInPlan, ok := b.is.TableByID(tblInfo.ID)
if !ok {
return nil, errors.Errorf("Can't get table %s.", tblInfo.Name.O)
}
return &SplitIndexRegion{
Table: tableInPlan,
IndexInfo: indexInfo,
ValueLists: indexValues,
}, nil

}

func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) {
var authErr error
switch v := node.(type) {
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.