Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

planner: fix wrong column offsets when processing dynamic pruning for IndexJoin #24659

Merged
merged 7 commits into from
May 17, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions executor/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import (
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/admin"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/dbterror"
"github.com/pingcap/tidb/util/execdetails"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/ranger"
Expand Down Expand Up @@ -2476,11 +2477,14 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin)
outerKeyCols[i] = v.OuterJoinKeys[i].Index
}
innerKeyCols := make([]int, len(v.InnerJoinKeys))
innerKeyColIDs := make([]int64, len(v.InnerJoinKeys))
for i := 0; i < len(v.InnerJoinKeys); i++ {
innerKeyCols[i] = v.InnerJoinKeys[i].Index
innerKeyColIDs[i] = v.InnerJoinKeys[i].ID
}
e.outerCtx.keyCols = outerKeyCols
e.innerCtx.keyCols = innerKeyCols
e.innerCtx.keyColIDs = innerKeyColIDs

outerHashCols, innerHashCols := make([]int, len(v.OuterHashKeys)), make([]int, len(v.InnerHashKeys))
for i := 0; i < len(v.OuterHashKeys); i++ {
Expand Down Expand Up @@ -2785,7 +2789,6 @@ func keyColumnsIncludeAllPartitionColumns(keyColumns []int, pe *tables.Partition
func prunePartitionForInnerExecutor(ctx sessionctx.Context, tbl table.Table, schema *expression.Schema, partitionInfo *plannercore.PartitionInfo,
lookUpContent []*indexJoinLookUpContent) (usedPartition []table.PhysicalTable, canPrune bool, contentPos []int64, err error) {
partitionTbl := tbl.(table.PartitionedTable)
locateKey := make([]types.Datum, schema.Len())
// TODO: condition based pruning can be do in advance.
condPruneResult, err := partitionPruning(ctx, partitionTbl, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames)
if err != nil {
Expand All @@ -2800,22 +2803,44 @@ func prunePartitionForInnerExecutor(ctx sessionctx.Context, tbl table.Table, sch
if err != nil {
return nil, false, nil, err
}

// recalculate key column offsets
if lookUpContent[0].keyColIDs == nil {
return nil, false, nil,
dbterror.ClassOptimizer.NewStd(mysql.ErrInternal).GenWithStack("cannot get column IDs when dynamic pruning")
}
keyColOffsets := make([]int, len(lookUpContent[0].keyColIDs))
for i, colID := range lookUpContent[0].keyColIDs {
offset := -1
for j, col := range partitionTbl.Cols() {
if colID == col.ID {
offset = j
break
}
}
if offset == -1 {
return nil, false, nil,
dbterror.ClassOptimizer.NewStd(mysql.ErrInternal).GenWithStack("invalid column offset when dynamic pruning")
}
keyColOffsets[i] = offset
}

offsetMap := make(map[int]bool)
for _, offset := range lookUpContent[0].keyCols {
for _, offset := range keyColOffsets {
offsetMap[offset] = true
}
for _, offset := range pe.ColumnOffset {
if _, ok := offsetMap[offset]; !ok {
logutil.BgLogger().Warn("can not runtime prune in index join")
return condPruneResult, false, nil, nil
}
}

locateKey := make([]types.Datum, len(partitionTbl.Cols()))
partitions := make(map[int64]table.PhysicalTable)
contentPos = make([]int64, len(lookUpContent))
for idx, content := range lookUpContent {
for i, date := range content.keys {
locateKey[content.keyCols[i]] = date
locateKey[keyColOffsets[i]] = date
}
p, err := partitionTbl.GetPartitionByRow(ctx, locateKey)
if err != nil {
Expand Down
10 changes: 6 additions & 4 deletions executor/index_lookup_join.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ type innerCtx struct {
readerBuilder *dataReaderBuilder
rowTypes []*types.FieldType
keyCols []int
keyColIDs []int64 // the original ID in its table, used by dynamic partition pruning
hashCols []int
colLens []int
hasPrefixCol bool
Expand Down Expand Up @@ -472,9 +473,10 @@ func (iw *innerWorker) run(ctx context.Context, wg *sync.WaitGroup) {
}

type indexJoinLookUpContent struct {
keys []types.Datum
row chunk.Row
keyCols []int
keys []types.Datum
row chunk.Row
keyCols []int
keyColIDs []int64 // the original ID in its table, used by dynamic partition pruning
}

func (iw *innerWorker) handleTask(ctx context.Context, task *lookUpJoinTask) error {
Expand Down Expand Up @@ -545,7 +547,7 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi
// dLookUpKey is sorted and deduplicated at sortAndDedupLookUpContents.
// So we don't need to do it here.
}
lookUpContents = append(lookUpContents, &indexJoinLookUpContent{keys: dLookUpKey, row: chk.GetRow(rowIdx), keyCols: iw.keyCols})
lookUpContents = append(lookUpContents, &indexJoinLookUpContent{keys: dLookUpKey, row: chk.GetRow(rowIdx), keyCols: iw.keyCols, keyColIDs: iw.keyColIDs})
}
}

Expand Down
59 changes: 59 additions & 0 deletions executor/partition_table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,65 @@ func (s *partitionTableSuite) TestView(c *C) {
}
}

func (s *partitionTableSuite) TestDynamicPruningUnderIndexJoin(c *C) {
if israce.RaceEnabled {
c.Skip("exhaustive types test, skip race test")
}

tk := testkit.NewTestKitWithInit(c, s.store)
tk.MustExec("create database pruing_under_index_join")
tk.MustExec("use pruing_under_index_join")
tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'")

tk.MustExec(`create table tnormal (a int, b int, c int, primary key(a), index idx_b(b))`)
tk.MustExec(`create table thash (a int, b int, c int, primary key(a), index idx_b(b)) partition by hash(a) partitions 4`)
tk.MustExec(`create table touter (a int, b int, c int)`)

vals := make([]string, 0, 2000)
for i := 0; i < 2000; i++ {
vals = append(vals, fmt.Sprintf("(%v, %v, %v)", i, rand.Intn(10000), rand.Intn(10000)))
}
tk.MustExec(`insert into tnormal values ` + strings.Join(vals, ", "))
tk.MustExec(`insert into thash values ` + strings.Join(vals, ", "))
tk.MustExec(`insert into touter values ` + strings.Join(vals, ", "))

// case 1: IndexReader in the inner side
tk.MustQuery(`explain format='brief' select /*+ INL_JOIN(touter, thash) */ thash.b from touter join thash use index(idx_b) on touter.b = thash.b`).Check(testkit.Rows(
`IndexJoin 12487.50 root inner join, inner:IndexReader, outer key:pruing_under_index_join.touter.b, inner key:pruing_under_index_join.thash.b, equal cond:eq(pruing_under_index_join.touter.b, pruing_under_index_join.thash.b)`,
`├─TableReader(Build) 9990.00 root data:Selection`,
`│ └─Selection 9990.00 cop[tikv] not(isnull(pruing_under_index_join.touter.b))`,
`│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`,
`└─IndexReader(Probe) 1.25 root partition:all index:Selection`,
` └─Selection 1.25 cop[tikv] not(isnull(pruing_under_index_join.thash.b))`,
` └─IndexRangeScan 1.25 cop[tikv] table:thash, index:idx_b(b) range: decided by [eq(pruing_under_index_join.thash.b, pruing_under_index_join.touter.b)], keep order:false, stats:pseudo`))
tk.MustQuery(`select /*+ INL_JOIN(touter, thash) */ thash.b from touter join thash use index(idx_b) on touter.b = thash.b`).Sort().Check(
tk.MustQuery(`select /*+ INL_JOIN(touter, tnormal) */ tnormal.b from touter join tnormal use index(idx_b) on touter.b = tnormal.b`).Sort().Rows())

// case 2: TableReader in the inner side
tk.MustQuery(`explain format='brief' select /*+ INL_JOIN(touter, thash) */ thash.* from touter join thash use index(primary) on touter.b = thash.a`).Check(testkit.Rows(
`IndexJoin 12487.50 root inner join, inner:TableReader, outer key:pruing_under_index_join.touter.b, inner key:pruing_under_index_join.thash.a, equal cond:eq(pruing_under_index_join.touter.b, pruing_under_index_join.thash.a)`,
`├─TableReader(Build) 9990.00 root data:Selection`,
`│ └─Selection 9990.00 cop[tikv] not(isnull(pruing_under_index_join.touter.b))`,
`│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`,
`└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan`,
` └─TableRangeScan 1.00 cop[tikv] table:thash range: decided by [pruing_under_index_join.touter.b], keep order:false, stats:pseudo`))
tk.MustQuery(`select /*+ INL_JOIN(touter, thash) */ thash.* from touter join thash use index(primary) on touter.b = thash.a`).Sort().Check(
tk.MustQuery(`select /*+ INL_JOIN(touter, tnormal) */ tnormal.* from touter join tnormal use index(primary) on touter.b = tnormal.a`).Sort().Rows())

// case 3: IndexLookUp in the inner side + read all inner columns
tk.MustQuery(`explain format='brief' select /*+ INL_JOIN(touter, thash) */ thash.* from touter join thash use index(idx_b) on touter.b = thash.b`).Check(testkit.Rows(
`IndexJoin 12487.50 root inner join, inner:IndexLookUp, outer key:pruing_under_index_join.touter.b, inner key:pruing_under_index_join.thash.b, equal cond:eq(pruing_under_index_join.touter.b, pruing_under_index_join.thash.b)`,
`├─TableReader(Build) 9990.00 root data:Selection`,
`│ └─Selection 9990.00 cop[tikv] not(isnull(pruing_under_index_join.touter.b))`,
`│ └─TableFullScan 10000.00 cop[tikv] table:touter keep order:false, stats:pseudo`,
`└─IndexLookUp(Probe) 1.25 root partition:all `,
` ├─Selection(Build) 1.25 cop[tikv] not(isnull(pruing_under_index_join.thash.b))`,
` │ └─IndexRangeScan 1.25 cop[tikv] table:thash, index:idx_b(b) range: decided by [eq(pruing_under_index_join.thash.b, pruing_under_index_join.touter.b)], keep order:false, stats:pseudo`,
` └─TableRowIDScan(Probe) 1.25 cop[tikv] table:thash keep order:false, stats:pseudo`))
tk.MustQuery(`select /*+ INL_JOIN(touter, thash) */ thash.* from touter join thash use index(idx_b) on touter.b = thash.b`).Sort().Check(
tk.MustQuery(`select /*+ INL_JOIN(touter, tnormal) */ tnormal.* from touter join tnormal use index(idx_b) on touter.b = tnormal.b`).Sort().Rows())
}

func (s *partitionTableSuite) TestGlobalStatsAndSQLBinding(c *C) {
if israce.RaceEnabled {
c.Skip("exhaustive types test, skip race test")
Expand Down