Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Planner:eliminate redundant projection #11920

Merged
merged 12 commits into from
Sep 2, 2019
Merged

Planner:eliminate redundant projection #11920

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
a3b3810
Planner:eliminate redundant projection
jingyugao Aug 28, 2019
a0c30c1
add test cases
jingyugao Aug 28, 2019
76ef790
generate explaintest
jingyugao Aug 28, 2019
6b5a19f
rm non-related change
jingyugao Aug 28, 2019
6cd138b
fix review comment
jingyugao Aug 29, 2019
d92867f
eliminater even exprs not equal
jingyugao Aug 29, 2019
3c24b45
fix review comment
jingyugao Aug 29, 2019
a5a2578
fix review comment
jingyugao Aug 29, 2019
05f028d
Merge branch 'master' into issue#11863
jingyugao Aug 31, 2019
6ea840f
dont eliminate if child has side-effect function
jingyugao Aug 31, 2019
a962f54
fix typo
jingyugao Aug 31, 2019
c15a43d
Merge branch 'master' into issue#11863
sre-bot Sep 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 16 additions & 20 deletions cmd/explaintest/r/explain_easy.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,13 @@ Projection_9 9990.00 root test.t1.c1
└─TableScan_18 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select (select count(1) k from t1 s where s.c1 = t1.c1 having k != 0) from t1;
id count task operator info
Projection_12 10000.00 root k
└─Projection_13 10000.00 root test.t1.c1, ifnull(5_col_0, 0)
└─MergeJoin_14 10000.00 root left outer join, left key:test.t1.c1, right key:test.s.c1
├─TableReader_17 10000.00 root data:TableScan_16
│ └─TableScan_16 10000.00 cop table:t1, range:[-inf,+inf], keep order:true, stats:pseudo
└─Projection_18 8000.00 root 1, test.s.c1
└─TableReader_20 10000.00 root data:TableScan_19
└─TableScan_19 10000.00 cop table:s, range:[-inf,+inf], keep order:true, stats:pseudo
Projection_12 10000.00 root ifnull(5_col_0, 0)
└─MergeJoin_13 10000.00 root left outer join, left key:test.t1.c1, right key:test.s.c1
├─TableReader_16 10000.00 root data:TableScan_15
│ └─TableScan_15 10000.00 cop table:t1, range:[-inf,+inf], keep order:true, stats:pseudo
└─Projection_17 8000.00 root 1, test.s.c1
└─TableReader_19 10000.00 root data:TableScan_18
└─TableScan_18 10000.00 cop table:s, range:[-inf,+inf], keep order:true, stats:pseudo
explain select * from information_schema.columns;
id count task operator info
MemTableScan_4 10000.00 root
Expand Down Expand Up @@ -648,22 +647,19 @@ id count task operator info
Sort_13 2.00 root a:asc
└─HashAgg_17 2.00 root group by:a, funcs:firstrow(join_agg_0)
└─Union_18 2.00 root
├─HashAgg_21 1.00 root group by:a, funcs:firstrow(a), firstrow(a)
│ └─Projection_22 1.00 root 0
│ └─TableDual_23 1.00 root rows:1
└─HashAgg_26 1.00 root group by:a, funcs:firstrow(a), firstrow(a)
└─Projection_27 1.00 root 1
└─TableDual_28 1.00 root rows:1
├─HashAgg_19 1.00 root group by:0, funcs:firstrow(0), firstrow(0)
│ └─TableDual_22 1.00 root rows:1
└─HashAgg_25 1.00 root group by:1, funcs:firstrow(1), firstrow(1)
└─TableDual_28 1.00 root rows:1
explain SELECT 0 AS a FROM dual UNION (SELECT 1 AS a FROM dual ORDER BY a);
id count task operator info
HashAgg_15 2.00 root group by:a, funcs:firstrow(join_agg_0)
└─Union_16 2.00 root
├─HashAgg_19 1.00 root group by:a, funcs:firstrow(a), firstrow(a)
│ └─Projection_20 1.00 root 0
│ └─TableDual_21 1.00 root rows:1
└─StreamAgg_26 1.00 root group by:a, funcs:firstrow(a), firstrow(a)
└─Projection_31 1.00 root 1
└─TableDual_32 1.00 root rows:1
├─HashAgg_17 1.00 root group by:0, funcs:firstrow(0), firstrow(0)
│ └─TableDual_20 1.00 root rows:1
└─StreamAgg_27 1.00 root group by:a, funcs:firstrow(a), firstrow(a)
└─Projection_32 1.00 root 1
└─TableDual_33 1.00 root rows:1
create table t (i int key, j int, unique key (i, j));
begin;
insert into t values (1, 1);
Expand Down
10 changes: 5 additions & 5 deletions cmd/explaintest/r/partition_pruning.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4010,11 +4010,11 @@ create table t1 (s1 int);
explain select 1 from t1 union all select 2;
id count task operator info
Union_8 10001.00 root
├─Projection_10 10000.00 root 1
│ └─TableReader_12 10000.00 root data:TableScan_11
│ └─TableScan_11 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo
└─Projection_14 1.00 root 2
└─TableDual_15 1.00 root rows:1
├─Projection_9 10000.00 root 1
│ └─TableReader_11 10000.00 root data:TableScan_10
│ └─TableScan_10 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo
└─Projection_12 1.00 root 2
└─TableDual_13 1.00 root rows:1
drop table t1;
create table t1 (a int)
partition by range(a) (
Expand Down
32 changes: 16 additions & 16 deletions cmd/explaintest/r/select.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,24 +382,24 @@ drop table if exists t;
create table t(a int, b int);
explain select a != any (select a from t t2) from t t1;
id count task operator info
Projection_9 10000.00 root and(or(or(gt(col_count, 1), ne(test.t1.a, col_firstrow)), if(ne(agg_col_sum, 0), NULL, 0)), and(ne(agg_col_cnt, 0), if(isnull(test.t1.a), NULL, 1)))
└─HashLeftJoin_10 10000.00 root CARTESIAN inner join, inner:StreamAgg_17
├─TableReader_13 10000.00 root data:TableScan_12
│ └─TableScan_12 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo
└─StreamAgg_17 1.00 root funcs:firstrow(col_0), count(distinct col_1), sum(col_2), count(1)
└─Projection_27 10000.00 root test.t2.a, test.t2.a, cast(isnull(test.t2.a))
└─TableReader_24 10000.00 root data:TableScan_23
└─TableScan_23 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
Projection_8 10000.00 root and(or(or(gt(col_count, 1), ne(test.t1.a, col_firstrow)), if(ne(agg_col_sum, 0), NULL, 0)), and(ne(agg_col_cnt, 0), if(isnull(test.t1.a), NULL, 1)))
└─HashLeftJoin_9 10000.00 root CARTESIAN inner join, inner:StreamAgg_16
├─TableReader_12 10000.00 root data:TableScan_11
│ └─TableScan_11 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo
└─StreamAgg_16 1.00 root funcs:firstrow(col_0), count(distinct col_1), sum(col_2), count(1)
└─Projection_26 10000.00 root test.t2.a, test.t2.a, cast(isnull(test.t2.a))
└─TableReader_23 10000.00 root data:TableScan_22
└─TableScan_22 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
explain select a = all (select a from t t2) from t t1;
id count task operator info
Projection_9 10000.00 root or(and(and(le(col_count, 1), eq(test.t1.a, col_firstrow)), if(ne(agg_col_sum, 0), NULL, 1)), or(eq(agg_col_cnt, 0), if(isnull(test.t1.a), NULL, 0)))
└─HashLeftJoin_10 10000.00 root CARTESIAN inner join, inner:StreamAgg_17
├─TableReader_13 10000.00 root data:TableScan_12
│ └─TableScan_12 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo
└─StreamAgg_17 1.00 root funcs:firstrow(col_0), count(distinct col_1), sum(col_2), count(1)
└─Projection_27 10000.00 root test.t2.a, test.t2.a, cast(isnull(test.t2.a))
└─TableReader_24 10000.00 root data:TableScan_23
└─TableScan_23 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
Projection_8 10000.00 root or(and(and(le(col_count, 1), eq(test.t1.a, col_firstrow)), if(ne(agg_col_sum, 0), NULL, 1)), or(eq(agg_col_cnt, 0), if(isnull(test.t1.a), NULL, 0)))
└─HashLeftJoin_9 10000.00 root CARTESIAN inner join, inner:StreamAgg_16
├─TableReader_12 10000.00 root data:TableScan_11
│ └─TableScan_11 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo
└─StreamAgg_16 1.00 root funcs:firstrow(col_0), count(distinct col_1), sum(col_2), count(1)
└─Projection_26 10000.00 root test.t2.a, test.t2.a, cast(isnull(test.t2.a))
└─TableReader_23 10000.00 root data:TableScan_22
└─TableScan_22 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
drop table if exists t;
create table t(a int, b int);
drop table if exists s;
Expand Down
9 changes: 4 additions & 5 deletions cmd/explaintest/r/topn_pushdown.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@ explain select * from ((select 4 as a) union all (select 33 as a)) tmp order by
id count task operator info
TopN_17 1.00 root tmp.a:desc, offset:0, count:1
└─Union_21 2.00 root
├─Projection_22 1.00 root cast(a)
│ └─Projection_23 1.00 root 4
│ └─TableDual_24 1.00 root rows:1
└─Projection_26 1.00 root 33
└─TableDual_27 1.00 root rows:1
├─Projection_22 1.00 root cast(4)
│ └─TableDual_23 1.00 root rows:1
└─Projection_24 1.00 root 33
└─TableDual_25 1.00 root rows:1
40 changes: 33 additions & 7 deletions planner/core/logical_plan_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1269,6 +1269,32 @@ func (s *testPlanSuite) TestColumnPruning(c *C) {
}
}

func (s *testPlanSuite) TestProjectionEliminater(c *C) {
defer testleak.AfterTest(c)()
tests := []struct {
sql string
best string
}{
{
sql: "select 1+num from (select 1+a as num from t) t1;",
best: "DataScan(t)->Projection",
},
}

ctx := context.Background()
for ith, tt := range tests {
comment := Commentf("for %s", tt.sql)
stmt, err := s.ParseOneStmt(tt.sql, "", "")
c.Assert(err, IsNil, comment)

p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is)
c.Assert(err, IsNil)
p, err = logicalOptimize(context.TODO(), flagBuildKeyInfo|flagPrunColumns|flagEliminateProjection, p.(LogicalPlan))
c.Assert(err, IsNil)
c.Assert(ToString(p), Equals, tt.best, Commentf("for %s %d", tt.sql, ith))
}
}

func (s *testPlanSuite) TestAllocID(c *C) {
ctx := MockContext()
pA := DataSource{}.Init(ctx)
Expand Down Expand Up @@ -1571,27 +1597,27 @@ func (s *testPlanSuite) TestAggPrune(c *C) {
}{
{
sql: "select a, count(b) from t group by a",
best: "DataScan(t)->Projection->Projection",
best: "DataScan(t)->Projection",
},
{
sql: "select sum(b) from t group by c, d, e",
best: "DataScan(t)->Aggr(sum(test.t.b))->Projection",
},
{
sql: "select tt.a, sum(tt.b) from (select a, b from t) tt group by tt.a",
best: "DataScan(t)->Projection->Projection",
best: "DataScan(t)->Projection",
},
{
sql: "select count(1) from (select count(1), a as b from t group by a) tt group by b",
best: "DataScan(t)->Projection->Projection",
best: "DataScan(t)->Projection",
},
{
sql: "select a, count(b) from t group by a",
best: "DataScan(t)->Projection->Projection",
best: "DataScan(t)->Projection",
},
{
sql: "select a, count(distinct a, b) from t group by a",
best: "DataScan(t)->Projection->Projection",
best: "DataScan(t)->Projection",
},
}

Expand Down Expand Up @@ -1897,12 +1923,12 @@ func (s *testPlanSuite) TestUnion(c *C) {
},
{
sql: "select * from (select 1 as a union select 1 union all select 2) t order by a",
best: "UnionAll{UnionAll{Dual->Projection->Projection->Dual->Projection->Projection}->Aggr(firstrow(a))->Projection->Dual->Projection->Projection}->Projection->Sort",
best: "UnionAll{UnionAll{Dual->Projection->Dual->Projection}->Aggr(firstrow(a))->Projection->Dual->Projection}->Projection->Sort",
err: false,
},
{
sql: "select * from (select 1 as a union select 1 union all select 2) t order by (select a)",
best: "Apply{UnionAll{UnionAll{Dual->Projection->Projection->Dual->Projection->Projection}->Aggr(firstrow(a))->Projection->Dual->Projection->Projection}->Dual->Projection->MaxOneRow}->Sort->Projection",
best: "Apply{UnionAll{UnionAll{Dual->Projection->Dual->Projection}->Aggr(firstrow(a))->Projection->Dual->Projection}->Dual->Projection->MaxOneRow}->Sort->Projection",
err: false,
},
}
Expand Down
4 changes: 2 additions & 2 deletions planner/core/physical_plan_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderSubquery(c *C) {
},
{
sql: "select (select count(*) from t s, t t1 where s.a = t.a and s.a = t1.a) from t",
best: "LeftHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->TableReader(Table(t))}(test.s.a,test.t1.a)->StreamAgg}(test.t.a,test.s.a)->Projection->Projection",
best: "LeftHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->TableReader(Table(t))}(test.s.a,test.t1.a)->StreamAgg}(test.t.a,test.s.a)->Projection",
},
{
sql: "select (select count(*) from t s, t t1 where s.a = t.a and s.a = t1.a) from t order by t.a",
Expand Down Expand Up @@ -912,7 +912,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderAgg(c *C) {
},
{
sql: "select (select count(1) k from t s where s.a = t.a having k != 0) from t",
best: "MergeLeftOuterJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t.a,test.s.a)->Projection->Projection",
best: "MergeLeftOuterJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t.a,test.s.a)->Projection",
},
// Test stream agg with multi group by columns.
{
Expand Down
24 changes: 24 additions & 0 deletions planner/core/rule_eliminate_projection.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,14 @@ func (pe *projectionEliminater) eliminate(p LogicalPlan, replace map[string]*exp
}
}
p.replaceExprColumns(replace)
if isProj {
if child, ok := p.Children()[0].(*LogicalProjection); ok {
for i := range proj.Exprs {
proj.Exprs[i] = replaceColumnOfExpr(proj.Exprs[i], child)
}
p.Children()[0] = child.Children()[0]
}
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we fix this issue by removing the constraint of canProjectionBeEliminatedLoose and enhancing replace map[string]*expression.Column to replace map[string]expression.Expression? we need to update resolveExprAndReplace though.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some operator will use replace map[string]*expression.Column to replace the schema.
I am not sure how to handle it.


if !(isProj && canEliminate && canProjectionBeEliminatedLoose(proj)) {
return p
Expand All @@ -148,6 +156,22 @@ func (pe *projectionEliminater) eliminate(p LogicalPlan, replace map[string]*exp
return p.Children()[0]
}

func replaceColumnOfExpr(expr expression.Expression, proj *LogicalProjection) expression.Expression {
switch v := expr.(type) {
case *expression.Column:
for i := range proj.Schema().Columns {
francis0407 marked this conversation as resolved.
Show resolved Hide resolved
if proj.Schema().Columns[i].Equal(proj.SCtx(), v) {
return proj.Exprs[i]
}
}
case *expression.ScalarFunction:
for i := range v.GetArgs() {
v.GetArgs()[i] = replaceColumnOfExpr(v.GetArgs()[i], proj)
}
}
return expr
}

func (p *LogicalJoin) replaceExprColumns(replace map[string]*expression.Column) {
for _, equalExpr := range p.EqualConditions {
resolveExprAndReplace(equalExpr, replace)
Expand Down