Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

planner, executor: support index merge's order prop push down at the normal way #43881

Merged
merged 16 commits into from Aug 1, 2023
Merged

planner, executor: support index merge's order prop push down at the normal way #43881

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
18b5cee
planner, executor: support index merge's order prop push down at the …
winoros May 16, 2023
5f8dc44
fix test
winoros May 16, 2023
a246875
add-generic-shallow-clone
winoros May 23, 2023
08c86e0
address comments
winoros May 23, 2023
fd75137
Merge branch 'master' into support-indexmerge-ordered-scan
winoros Jul 18, 2023
c1da5cd
add some changes to align with master
winoros Jul 18, 2023
880da73
update tests
winoros Jul 18, 2023
2c3c27c
add the case that only order or only limit is pushed
winoros Jul 18, 2023
9b86d70
Merge branch 'master' into support-indexmerge-ordered-scan
winoros Jul 25, 2023
6013468
address comments
winoros Jul 25, 2023
586d581
Apply suggestions from code review
winoros Jul 31, 2023
7f0d0ae
address comments
winoros Jul 31, 2023
9930a4f
fix the problem of dirty read
winoros Jul 31, 2023
b97ed79
add test for dirty read
winoros Jul 31, 2023
7cb3795
Merge branch 'master' into support-indexmerge-ordered-scan
winoros Jul 31, 2023
ebfc8f4
Merge branch 'master' into support-indexmerge-ordered-scan
winoros Jul 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion executor/builder.go
View file
Open in desktop
Expand Up @@ -4121,7 +4121,7 @@ func buildNoRangeIndexMergeReader(b *executorBuilder, v *plannercore.PhysicalInd
dataReaderBuilder: readerBuilder,
feedbacks: feedbacks,
paging: paging,
handleCols: ts.HandleCols,
handleCols: v.HandleCols,
isCorColInPartialFilters: isCorColInPartialFilters,
isCorColInTableFilter: isCorColInTableFilter,
isCorColInPartialAccess: isCorColInPartialAccess,
Expand Down
8 changes: 4 additions & 4 deletions executor/index_merge_reader.go
View file
Open in desktop
Expand Up @@ -338,6 +338,7 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
util.WithRecovery(
func() {
failpoint.Inject("testIndexMergePanicPartialIndexWorker", nil)
is := e.partialPlans[workID][0].(*plannercore.PhysicalIndexScan)
worker := &partialIndexWorker{
stats: e.stats,
idxID: e.getPartitalPlanID(workID),
Expand All @@ -348,10 +349,9 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
memTracker: e.memTracker,
partitionTableMode: e.partitionTableMode,
prunedPartitions: e.prunedPartitions,
byItems: e.byItems,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like we don't need to change here.
Because we will set this in PhysicalIndexMergeReader.ByItems (Init()) then IndexMergeReaderExecutor.byItems (buildNoRangeIndexMergeReader), so finally they are the same.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I want to do the remained part in the later pull.

byItems: is.ByItems,
pushedLimit: e.pushedLimit,
}

if e.isCorColInPartialFilters[workID] {
// We got correlated column, so need to refresh Selection operator.
var err error
Expand Down Expand Up @@ -466,7 +466,7 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
ranges: e.ranges[workID],
netDataSize: e.partialNetDataSizes[workID],
keepOrder: ts.KeepOrder,
byItems: e.byItems,
byItems: ts.ByItems,
}

worker := &partialTableWorker{
Expand All @@ -479,7 +479,7 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
memTracker: e.memTracker,
partitionTableMode: e.partitionTableMode,
prunedPartitions: e.prunedPartitions,
byItems: e.byItems,
byItems: ts.ByItems,
pushedLimit: e.pushedLimit,
}

Expand Down
78 changes: 39 additions & 39 deletions planner/core/casetest/physicalplantest/testdata/plan_suite_out.json
View file
Open in desktop
Expand Up @@ -8497,13 +8497,13 @@
{
"SQL": "select * from t where a = 1 or b = 1 order by c limit 2",
"Plan": [
"Projection 19.99 root test.t.a, test.t.b, test.t.c",
"└─IndexMerge 19.99 root type: union, limit embedded(offset:0, count:2)",
"Projection 2.00 root test.t.a, test.t.b, test.t.c",
"└─IndexMerge 2.00 root type: union, limit embedded(offset:0, count:2)",
" ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 2.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], keep order:true, stats:pseudo",
" │ └─IndexRangeScan 10.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], keep order:true, stats:pseudo",
" ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 2.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
" └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t keep order:false, stats:pseudo"
" │ └─IndexRangeScan 10.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
" └─TableRowIDScan(Probe) 2.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
Expand Down Expand Up @@ -8546,13 +8546,13 @@
{
"SQL": "select * from t where (a = 1 and c = 2) or (b = 1) order by c limit 2",
"Plan": [
"Projection 10.10 root test.t.a, test.t.b, test.t.c",
"└─IndexMerge 10.10 root type: union, limit embedded(offset:0, count:2)",
"Projection 2.00 root test.t.a, test.t.b, test.t.c",
"└─IndexMerge 2.00 root type: union, limit embedded(offset:0, count:2)",
" ├─Limit(Build) 0.10 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 0.10 cop[tikv] table:t, index:idx(a, c) range:[1 2,1 2], keep order:true, stats:pseudo",
" ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 2.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
" └─TableRowIDScan(Probe) 10.10 cop[tikv] table:t keep order:false, stats:pseudo"
" │ └─IndexRangeScan 10.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
" └─TableRowIDScan(Probe) 2.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
Expand Down Expand Up @@ -8606,24 +8606,24 @@
{
"SQL": "select * from tcommon where a = 1 or b = 1 order by c limit 2",
"Plan": [
"IndexMerge 19.99 root type: union, limit embedded(offset:0, count:2)",
"├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
"│ └─TableRangeScan 2.00 cop[tikv] table:tcommon range:[1,1], keep order:true, stats:pseudo",
"IndexMerge 2.00 root type: union, limit embedded(offset:0, count:2)",
"├─Limit(Build) 1.00 cop[tikv] offset:0, count:2",
"│ └─TableRangeScan 1.00 cop[tikv] table:tcommon range:[1,1], keep order:true, stats:pseudo",
"├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
"│ └─IndexRangeScan 2.00 cop[tikv] table:tcommon, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
"└─TableRowIDScan(Probe) 19.99 cop[tikv] table:tcommon keep order:false, stats:pseudo"
"│ └─IndexRangeScan 10.00 cop[tikv] table:tcommon, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
"└─TableRowIDScan(Probe) 2.00 cop[tikv] table:tcommon keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from tcommon where (a = 1 and c = 2) or (b = 1) order by c limit 2",
"Plan": [
"IndexMerge 11.00 root type: union, limit embedded(offset:0, count:2)",
"├─Limit(Build) 1.00 cop[tikv] offset:0, count:2",
"│ └─TableRangeScan 1.00 cop[tikv] table:tcommon range:[1 2,1 2], keep order:true, stats:pseudo",
"IndexMerge 2.00 root type: union, limit embedded(offset:0, count:2)",
"├─Limit(Build) 0.18 cop[tikv] offset:0, count:2",
"│ └─TableRangeScan 0.18 cop[tikv] table:tcommon range:[1 2,1 2], keep order:true, stats:pseudo",
"├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
"│ └─IndexRangeScan 2.00 cop[tikv] table:tcommon, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
"└─TableRowIDScan(Probe) 11.00 cop[tikv] table:tcommon keep order:false, stats:pseudo"
"│ └─IndexRangeScan 10.00 cop[tikv] table:tcommon, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
"└─TableRowIDScan(Probe) 2.00 cop[tikv] table:tcommon keep order:false, stats:pseudo"
],
"Warning": null
},
Expand All @@ -8632,34 +8632,34 @@
"Plan": [
"TopN 2.00 root test.thash.c, offset:0, count:2",
"└─PartitionUnion 8.00 root ",
" ├─Projection 19.99 root test.thash.a, test.thash.b, test.thash.c",
" │ └─IndexMerge 19.99 root type: union, limit embedded(offset:0, count:2)",
" ├─Projection 2.00 root test.thash.a, test.thash.b, test.thash.c",
" │ └─IndexMerge 2.00 root type: union, limit embedded(offset:0, count:2)",
" │ ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ │ └─IndexRangeScan 2.00 cop[tikv] table:thash, partition:p0, index:idx_ac(a, c) range:[1,1], keep order:true, stats:pseudo",
" │ │ └─IndexRangeScan 10.00 cop[tikv] table:thash, partition:p0, index:idx_ac(a, c) range:[1,1], keep order:true, stats:pseudo",
" │ ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ │ └─IndexRangeScan 2.00 cop[tikv] table:thash, partition:p0, index:idx_bc(b, c) range:[1,1], keep order:true, stats:pseudo",
" │ └─TableRowIDScan(Probe) 19.99 cop[tikv] table:thash, partition:p0 keep order:false, stats:pseudo",
" ├─Projection 19.99 root test.thash.a, test.thash.b, test.thash.c",
" │ └─IndexMerge 19.99 root type: union, limit embedded(offset:0, count:2)",
" │ │ └─IndexRangeScan 10.00 cop[tikv] table:thash, partition:p0, index:idx_bc(b, c) range:[1,1], keep order:true, stats:pseudo",
" │ └─TableRowIDScan(Probe) 2.00 cop[tikv] table:thash, partition:p0 keep order:false, stats:pseudo",
" ├─Projection 2.00 root test.thash.a, test.thash.b, test.thash.c",
" │ └─IndexMerge 2.00 root type: union, limit embedded(offset:0, count:2)",
" │ ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ │ └─IndexRangeScan 2.00 cop[tikv] table:thash, partition:p1, index:idx_ac(a, c) range:[1,1], keep order:true, stats:pseudo",
" │ │ └─IndexRangeScan 10.00 cop[tikv] table:thash, partition:p1, index:idx_ac(a, c) range:[1,1], keep order:true, stats:pseudo",
" │ ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ │ └─IndexRangeScan 2.00 cop[tikv] table:thash, partition:p1, index:idx_bc(b, c) range:[1,1], keep order:true, stats:pseudo",
" │ └─TableRowIDScan(Probe) 19.99 cop[tikv] table:thash, partition:p1 keep order:false, stats:pseudo",
" ├─Projection 19.99 root test.thash.a, test.thash.b, test.thash.c",
" │ └─IndexMerge 19.99 root type: union, limit embedded(offset:0, count:2)",
" │ │ └─IndexRangeScan 10.00 cop[tikv] table:thash, partition:p1, index:idx_bc(b, c) range:[1,1], keep order:true, stats:pseudo",
" │ └─TableRowIDScan(Probe) 2.00 cop[tikv] table:thash, partition:p1 keep order:false, stats:pseudo",
" ├─Projection 2.00 root test.thash.a, test.thash.b, test.thash.c",
" │ └─IndexMerge 2.00 root type: union, limit embedded(offset:0, count:2)",
" │ ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ │ └─IndexRangeScan 2.00 cop[tikv] table:thash, partition:p2, index:idx_ac(a, c) range:[1,1], keep order:true, stats:pseudo",
" │ │ └─IndexRangeScan 10.00 cop[tikv] table:thash, partition:p2, index:idx_ac(a, c) range:[1,1], keep order:true, stats:pseudo",
" │ ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ │ └─IndexRangeScan 2.00 cop[tikv] table:thash, partition:p2, index:idx_bc(b, c) range:[1,1], keep order:true, stats:pseudo",
" │ └─TableRowIDScan(Probe) 19.99 cop[tikv] table:thash, partition:p2 keep order:false, stats:pseudo",
" └─Projection 19.99 root test.thash.a, test.thash.b, test.thash.c",
" └─IndexMerge 19.99 root type: union, limit embedded(offset:0, count:2)",
" │ │ └─IndexRangeScan 10.00 cop[tikv] table:thash, partition:p2, index:idx_bc(b, c) range:[1,1], keep order:true, stats:pseudo",
" │ └─TableRowIDScan(Probe) 2.00 cop[tikv] table:thash, partition:p2 keep order:false, stats:pseudo",
" └─Projection 2.00 root test.thash.a, test.thash.b, test.thash.c",
" └─IndexMerge 2.00 root type: union, limit embedded(offset:0, count:2)",
" ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 2.00 cop[tikv] table:thash, partition:p3, index:idx_ac(a, c) range:[1,1], keep order:true, stats:pseudo",
" │ └─IndexRangeScan 10.00 cop[tikv] table:thash, partition:p3, index:idx_ac(a, c) range:[1,1], keep order:true, stats:pseudo",
" ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 2.00 cop[tikv] table:thash, partition:p3, index:idx_bc(b, c) range:[1,1], keep order:true, stats:pseudo",
" └─TableRowIDScan(Probe) 19.99 cop[tikv] table:thash, partition:p3 keep order:false, stats:pseudo"
" │ └─IndexRangeScan 10.00 cop[tikv] table:thash, partition:p3, index:idx_bc(b, c) range:[1,1], keep order:true, stats:pseudo",
" └─TableRowIDScan(Probe) 2.00 cop[tikv] table:thash, partition:p3 keep order:false, stats:pseudo"
],
"Warning": [
"Warning 1105 disable dynamic pruning due to thash has no global stats"
Expand Down