From 3ed0aded229461d4d8c380a2b0a3a698e0662991 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Mon, 24 Nov 2014 11:31:47 +0800 Subject: [PATCH 1/4] HAWQ-161. Port GPDB planner fixes to HAWQ Query crashed with segmentation fault with optimizer=off. The crash happened in executor, while the root cause is in planner. The planner mistakenly believe that, when both the root node of subplan and the corresponding main plan node are executed in QD, there is no need to call function ParallelizeCorrelatedSubPlan. While, actually, in this situation presented in the JIRA, both the main plan node Window and subplan node Agg are executed in QD, we still need to do the subplan parallelization, otherwise, the SeqScan node cannot get the parameter from main plan, because they are in different slices. --- src/backend/cdb/cdbllize.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/backend/cdb/cdbllize.c b/src/backend/cdb/cdbllize.c index 4aa8795c77..8e02f3c7a3 100644 --- a/src/backend/cdb/cdbllize.c +++ b/src/backend/cdb/cdbllize.c @@ -329,6 +329,7 @@ typedef struct ParallelizeCorrelatedPlanWalkerContext Movement movement; /* What is the final movement necessary? Is it gather or broadcast */ List *rtable; /* rtable from the global context */ bool subPlanDistributed; /* is original subplan distributed */ + bool subPlanHasMotion;/* is original subplan has motion already */ } ParallelizeCorrelatedPlanWalkerContext; /** @@ -462,6 +463,20 @@ static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelat || IsA(node, ShareInputScan)) { Plan *scanPlan = (Plan *) node; + /** + * If original subplan has no motion, we double check whether the scan + * node is on catalog table or not. If catalog, no need to apply + * parallelization. + * This is for case like: + * SELECT array(select case when p.oid in (select + * unnest(array[typoutput, typsend]) from pg_type) then 'upg_catalog.' + * else 'pg_catalog.' end) FROM pg_proc p; + **/ + if(!ctx->subPlanHasMotion) + { + if(!scanPlan->flow || scanPlan->flow->flotype == FLOW_REPLICATED) + return (Node *)node; + } /** * Steps: @@ -595,6 +610,19 @@ static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelat { return node; } + + /** + * Pull up the parallelization of subplan to here, because we want to + * set the is_parallelized flag. We don't want to apply double + * parallelization for nested subplan. + * This is for case like: + * select A.i, A.j, (select sum(C.j) from C where C.j = A.j and C.i = + * (select A.i from A where A.i = C.i)) from A; + **/ + SubPlan* new_sp = (SubPlan *)plan_tree_mutator(node, ParallelizeCorrelatedSubPlanMutator, ctx); + Assert(new_sp); + new_sp->is_parallelized = true; + return (Node *)new_sp; } /** @@ -603,6 +631,7 @@ static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelat */ if (IsA(node, Motion)) { + ctx->subPlanHasMotion = true; Plan *plan = (Plan *) node; node = (Node *) plan->lefttree; Assert(node); @@ -621,6 +650,7 @@ Plan* ParallelizeCorrelatedSubPlan(PlannerInfo *root, SubPlan *spExpr, Plan *pla ctx.base.node = (Node *) root; ctx.movement = m; ctx.subPlanDistributed = subPlanDistributed; + ctx.subPlanHasMotion = false; ctx.sp = spExpr; ctx.rtable = root->glob->finalrtable; return (Plan *) ParallelizeCorrelatedSubPlanMutator((Node *) plan, &ctx); @@ -655,6 +685,7 @@ void ParallelizeSubplan(SubPlan *spExpr, PlanProfile *context) bool containingPlanDistributed = (context->currentPlanFlow && context->currentPlanFlow->flotype == FLOW_PARTITIONED); bool subPlanDistributed = (origPlan->flow && origPlan->flow->flotype == FLOW_PARTITIONED); + bool hasParParam = (list_length(spExpr->parParam) > 0); /** * If containing plan is distributed then we must know the flow of the subplan. @@ -693,7 +724,11 @@ void ParallelizeSubplan(SubPlan *spExpr, PlanProfile *context) newPlan = materialize_subplan(context->root, newPlan); } - else if (containingPlanDistributed || subPlanDistributed) + /* * + * [JIRA: MPP-24563] Adding hasParParam check here, for the kind of cases in + * JIRA, which has both focused parent plan and subplan. + * */ + else if(containingPlanDistributed || subPlanDistributed || hasParParam) { Movement reqMove = containingPlanDistributed ? MOVEMENT_BROADCAST : MOVEMENT_FOCUS; From d9ce9c5f3c9258a367c814633510b2da06db7db1 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Wed, 24 Dec 2014 14:17:24 +0800 Subject: [PATCH 2/4] HAWQ-161. Port GPDB planner fixes to HAWQ Query Crashes with "Error on receive...server closed the connection unexpectedly". Same issue as MPP-24563; after applying the fix for MPP-24563, this case still failed because of assert failure in apply_shareinput_xslice. Root cause is that cdbparallelize changes the plan but does not update the flow of the upper nodes, therefore, once ParallelizeCorrelatedSubPlanMutator takes action, then we cannot gurantee the correctness of the upper nodes' flow. Fix is to update the flow after transforming the lower nodes. Note that, the function ParallelizeCorrelatedSubPlanUpdateFlowMutator currently only handles the three known situations, if more cases of this kind found, just twickle that function can solve the problem. --- src/backend/cdb/cdbllize.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/backend/cdb/cdbllize.c b/src/backend/cdb/cdbllize.c index 8e02f3c7a3..4d58b6ba9c 100644 --- a/src/backend/cdb/cdbllize.c +++ b/src/backend/cdb/cdbllize.c @@ -350,6 +350,7 @@ static void ParallelizeSubplan(SubPlan *spExpr, PlanProfile *context); static Plan* ParallelizeCorrelatedSubPlan(PlannerInfo *root, SubPlan *spExpr, Plan *plan, Movement m, bool subPlanDistributed); static Node* MapVarsMutator(Node *expr, MapVarsMutatorContext *ctx); static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelatedPlanWalkerContext *ctx); +static Node *ParallelizeCorrelatedSubPlanUpdateFlowMutator(Node *node); /** * Does an expression contain a parameter? @@ -415,6 +416,34 @@ Plan *materialize_subplan(PlannerInfo *root, Plan *subplan) return mat; } +static Node *ParallelizeCorrelatedSubPlanUpdateFlowMutator(Node *node) +{ + Assert(is_plan_node(node)); + switch (nodeTag(node)) + { + case T_Agg: + case T_Result: + { + if(((Plan *)node)->lefttree && ((Plan *)node)->lefttree->flow) + ((Plan *)node)->flow = pull_up_Flow((Plan *)node, ((Plan *)node)->lefttree, true); + break; + } + case T_Append: + { + List *append_list = (List *) ((Append *)node)->appendplans; + if(append_list == NULL) + break; + Plan *first_append = (Plan *) (lfirst(list_head(append_list))); + Assert(first_append && first_append->flow); + bool with_sort = list_length(append_list) == 1 ? true : false; + ((Plan *)node)->flow = pull_up_Flow((Plan *)node, first_append, with_sort); + break; + } + default: + break; + } + return node; +} /** * This is the workhorse method that transforms a plan containing correlated references @@ -587,6 +616,9 @@ static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelat ((Plan *) res)->allParam = saveAllParam; ((Plan *) res)->extParam = saveExtParam; + Assert(((Plan *) res)->lefttree && ((Plan *) res)->lefttree->flow); + ((Plan *) res)->flow = pull_up_Flow((Plan *) res, ((Plan *)res)->lefttree, true); + /** * It is possible that there is an additional level of correlation in the result node. * we will need to recurse in these structures again. @@ -638,7 +670,10 @@ static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelat return ParallelizeCorrelatedSubPlanMutator(node, ctx); } - return plan_tree_mutator(node, ParallelizeCorrelatedSubPlanMutator, ctx); + Node * result = plan_tree_mutator(node, ParallelizeCorrelatedSubPlanMutator, ctx); + if(is_plan_node(node)) + return ParallelizeCorrelatedSubPlanUpdateFlowMutator(result); + return result; } /** From 52a66665b9355cd1f6a422c78481ef05e466898e Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Thu, 12 Nov 2015 15:13:16 -0800 Subject: [PATCH 3/4] HAWQ-161. Port GPDB planner fixes to HAWQ Planner generated unnecessary motion node for nested subplan which scan catalog table. --- src/backend/cdb/cdbllize.c | 21 +++++++++++++-------- src/backend/optimizer/plan/setrefs.c | 1 + 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/backend/cdb/cdbllize.c b/src/backend/cdb/cdbllize.c index 4d58b6ba9c..59a59d8a38 100644 --- a/src/backend/cdb/cdbllize.c +++ b/src/backend/cdb/cdbllize.c @@ -329,7 +329,6 @@ typedef struct ParallelizeCorrelatedPlanWalkerContext Movement movement; /* What is the final movement necessary? Is it gather or broadcast */ List *rtable; /* rtable from the global context */ bool subPlanDistributed; /* is original subplan distributed */ - bool subPlanHasMotion;/* is original subplan has motion already */ } ParallelizeCorrelatedPlanWalkerContext; /** @@ -416,12 +415,20 @@ Plan *materialize_subplan(PlannerInfo *root, Plan *subplan) return mat; } +/* * + * Not listing every node type here, if further bug found due to not updating flow + * after cdbparallelize, just simply add that node type here. + * */ static Node *ParallelizeCorrelatedSubPlanUpdateFlowMutator(Node *node) { Assert(is_plan_node(node)); switch (nodeTag(node)) { case T_Agg: + case T_Window: + case T_Sort: + case T_Material: + case T_Limit: case T_Result: { if(((Plan *)node)->lefttree && ((Plan *)node)->lefttree->flow) @@ -501,11 +508,8 @@ static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelat * unnest(array[typoutput, typsend]) from pg_type) then 'upg_catalog.' * else 'pg_catalog.' end) FROM pg_proc p; **/ - if(!ctx->subPlanHasMotion) - { - if(!scanPlan->flow || scanPlan->flow->flotype == FLOW_REPLICATED) - return (Node *)node; - } + if(scanPlan->flow && (scanPlan->flow->locustype == CdbLocusType_Entry)) + return (Node *)node; /** * Steps: @@ -663,7 +667,6 @@ static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelat */ if (IsA(node, Motion)) { - ctx->subPlanHasMotion = true; Plan *plan = (Plan *) node; node = (Node *) plan->lefttree; Assert(node); @@ -671,6 +674,9 @@ static Node* ParallelizeCorrelatedSubPlanMutator(Node *node, ParallelizeCorrelat } Node * result = plan_tree_mutator(node, ParallelizeCorrelatedSubPlanMutator, ctx); + /* * + * Update the flow of the current plan node. + * */ if(is_plan_node(node)) return ParallelizeCorrelatedSubPlanUpdateFlowMutator(result); return result; @@ -685,7 +691,6 @@ Plan* ParallelizeCorrelatedSubPlan(PlannerInfo *root, SubPlan *spExpr, Plan *pla ctx.base.node = (Node *) root; ctx.movement = m; ctx.subPlanDistributed = subPlanDistributed; - ctx.subPlanHasMotion = false; ctx.sp = spExpr; ctx.rtable = root->glob->finalrtable; return (Plan *) ParallelizeCorrelatedSubPlanMutator((Node *) plan, &ctx); diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index afce5a598a..2f70516b30 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -2528,6 +2528,7 @@ cdb_insert_result_node(PlannerGlobal *glob, Plan *plan, int rtoffset) /* Reattach the Flow node. */ resultplan->flow = flow; + plan->flow = flow; return resultplan; } /* cdb_insert_result_node */ From 1cd375c3eedd66ea7cf21ce583c12311d84e5509 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Thu, 12 Nov 2015 15:36:37 -0800 Subject: [PATCH 4/4] HAWQ-161. Port GPDB planner fixes to HAWQ SIGSEGV happened in HashJoin. This issue was caused by a previous fix for MPP-12637, and therefore a wrong plan was generated for query like: select c.relname from pg_class c where c.relname in (select b.table_name from information_schema.tables b where c.relname like '%id%'); The RC for MPP-25724 is that, planner believes the subquery should not be pulled up because it contains CoerceToDomain node, while actually the subquery MUST be pulled up, otherwise HashJoin would fail. However, after I make planner pull up that subquery, the problem in MPP-12637 would happen again, i.e, for query like below, planner would error out. select * FROM (select attnum::information_schema.cardinal_number from pg_attribute where attnum > 0) q where attnum = 4; The key problem here is the order of quals, including the pulled up ones, i.e, the order of "attnum = 4" and "attnum > 0", my solution is replacing RangeTblRef of pulled up subquery with the resulting FromExpr, which is consistent with PostgreSQL's style, instead of separately appending the fromlist and quals of the resulting FromExpr to the parent query in previous GPDB. The final problem here now is in function deconstruct_jointree(). Previous assumption is that the quals must be compatible with the same level relids, however for query #1 mentioned above, we should break this assumption, so I introduced the PostponedQual feature of PostgreSQL. Meanwhile, we should make some corresponding changes in pull_up_simple_query to appropriately do the ResolveNew procedure now. --- src/backend/optimizer/path/pathkeys.c | 3 +- src/backend/optimizer/plan/initsplan.c | 143 +++++++++++++++--- src/backend/optimizer/prep/prepjointree.c | 55 ++----- src/include/optimizer/planmain.h | 3 +- src/test/regress/expected/gp_optimizer.out | 76 ++++++++++ .../regress/expected/information_schema.out | 34 +++++ .../expected/information_schema_optimizer.out | 34 +++++ src/test/regress/sql/gp_optimizer.sql | 63 ++++++++ src/test/regress/sql/information_schema.sql | 22 +++ 9 files changed, 371 insertions(+), 62 deletions(-) diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 3276b67ec3..f8c096a902 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -443,7 +443,8 @@ static List *gen_implied_qual(PlannerInfo *root, new_qualscope, /* qualscope */ NULL, /* ojscope */ NULL, /* outerjoin_nonnullable */ - NULL /* local equi join scope */ + NULL, /* local equi join scope */ + NULL /* postponed_qual_list */ ); relevant_clauses = lappend(relevant_clauses, new_clause); } diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index b3c726ef0c..18e34c8896 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -55,10 +55,20 @@ int from_collapse_limit; int join_collapse_limit; +/* a structure to be used in deconstruct_recurse(), records the qual clauses + * which cannot be distributed to specific relations immediately at that recurse + * level */ +typedef struct PostponedQual +{ + Node *qual; /* a qual clause waiting to be processed */ + Relids relids; /* the set of baserels it references */ +} PostponedQual; + static List *deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, Relids *qualscope, Relids *inner_join_rels, - List **ptrToLocalEquiKeyList); + List **ptrToLocalEquiKeyList, + List **postponed_qual_list); static OuterJoinInfo *make_outerjoininfo(PlannerInfo *root, Relids left_rels, Relids right_rels, Relids inner_join_rels, @@ -228,15 +238,24 @@ add_IN_vars_to_tlists(PlannerInfo *root) List * deconstruct_jointree(PlannerInfo *root) { + List *result = NIL; Relids qualscope; Relids inner_join_rels; + List *postponed_qual_list = NIL; /* Start recursion at top of jointree */ Assert(root->parse->jointree != NULL && IsA(root->parse->jointree, FromExpr)); - return deconstruct_recurse(root, (Node *) root->parse->jointree, false, - &qualscope, &inner_join_rels, NULL); + result = deconstruct_recurse(root, (Node *) root->parse->jointree, false, + &qualscope, &inner_join_rels, NULL, &postponed_qual_list); + + if (postponed_qual_list != NIL) + { + elog(ERROR, "JOIN qualification may not refer to other relations."); + } + + return result; } /* @@ -259,13 +278,16 @@ deconstruct_jointree(PlannerInfo *root) * values under the nullable side of an outer join are local equikeys * but not global equikeys) * Return value is the appropriate joinlist for this jointree node + * *postponed_qual_list gets the list of qual clauses which cannot be + * distributed to relids of current recurse level, and should be postponed to + * upper level to be handled * * In addition, entries will be added to root->oj_info_list for outer joins. */ static List * deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, Relids *qualscope, Relids *inner_join_rels, - List **ptrToLocalEquiKeyList) + List **ptrToLocalEquiKeyList, List **postponed_qual_list) { List *joinlist; @@ -290,6 +312,7 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, FromExpr *f = (FromExpr *) jtnode; int remaining; ListCell *l; + List *child_postponed_quals = NIL; /* * First, recurse to handle child joins. We collapse subproblems into @@ -311,7 +334,8 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, below_outer_join, &sub_qualscope, inner_join_rels, - ptrToLocalEquiKeyList); + ptrToLocalEquiKeyList, + &child_postponed_quals); *qualscope = bms_add_members(*qualscope, sub_qualscope); sub_members = list_length(sub_joinlist); remaining--; @@ -332,6 +356,31 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, if (list_length(f->fromlist) > 1) *inner_join_rels = *qualscope; + /* Try to process any quals postponed by children. If they need further + * postponement, add them to my output postponed_qual_list */ + foreach(l, child_postponed_quals) + { + PostponedQual *pq = (PostponedQual *) lfirst(l); + + if (bms_is_subset(pq->relids, *qualscope)) + { + distribute_qual_to_rels(root, pq->qual, + false, false, below_outer_join, + *qualscope, NULL, NULL, + ptrToLocalEquiKeyList, + NULL); + pfree(pq); + } + else + { + *postponed_qual_list = lappend(*postponed_qual_list, pq); + } + } + if (child_postponed_quals != NIL) + { + pfree(child_postponed_quals); + } + /* * Now process the top-level quals. */ @@ -339,7 +388,8 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, distribute_qual_to_rels(root, (Node *) lfirst(l), false, false, below_outer_join, *qualscope, NULL, NULL, - ptrToLocalEquiKeyList); + ptrToLocalEquiKeyList, + postponed_qual_list); } else if (IsA(jtnode, JoinExpr)) { @@ -358,6 +408,8 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, List *localLeftEquiKeyList = NIL; List *localRightEquiKeyList = NIL; + + List *child_postponed_quals = NIL; /* * Order of operations here is subtle and critical. First we recurse * to handle sub-JOINs. Their join quals will be placed without @@ -376,11 +428,13 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, leftjoinlist = deconstruct_recurse(root, j->larg, below_outer_join, &leftids, &left_inners, - ptrToLocalEquiKeyList); + ptrToLocalEquiKeyList, + &child_postponed_quals); rightjoinlist = deconstruct_recurse(root, j->rarg, below_outer_join, &rightids, &right_inners, - ptrToLocalEquiKeyList); + ptrToLocalEquiKeyList, + &child_postponed_quals); *qualscope = bms_union(leftids, rightids); *inner_join_rels = bms_copy(*qualscope); /* Inner join adds no restrictions for quals */ @@ -392,11 +446,13 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, leftjoinlist = deconstruct_recurse(root, j->larg, below_outer_join, &leftids, &left_inners, - ptrToLocalEquiKeyList); + ptrToLocalEquiKeyList, + &child_postponed_quals); rightjoinlist = deconstruct_recurse(root, j->rarg, true, &rightids, &right_inners, - &localRightEquiKeyList); + &localRightEquiKeyList, + &child_postponed_quals); *qualscope = bms_union(leftids, rightids); *inner_join_rels = bms_union(left_inners, right_inners); nonnullable_rels = leftids; @@ -405,11 +461,13 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, leftjoinlist = deconstruct_recurse(root, j->larg, true, &leftids, &left_inners, - &localLeftEquiKeyList); + &localLeftEquiKeyList, + &child_postponed_quals); rightjoinlist = deconstruct_recurse(root, j->rarg, true, &rightids, &right_inners, - &localRightEquiKeyList); + &localRightEquiKeyList, + &child_postponed_quals); *qualscope = bms_union(leftids, rightids); *inner_join_rels = bms_union(left_inners, right_inners); /* each side is both outer and inner */ @@ -420,11 +478,13 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, leftjoinlist = deconstruct_recurse(root, j->larg, true, &rightids, &right_inners, - &localRightEquiKeyList); + &localRightEquiKeyList, + &child_postponed_quals); rightjoinlist = deconstruct_recurse(root, j->rarg, below_outer_join, &leftids, &left_inners, - ptrToLocalEquiKeyList); + ptrToLocalEquiKeyList, + &child_postponed_quals); *qualscope = bms_union(leftids, rightids); *inner_join_rels = bms_union(left_inners, right_inners); nonnullable_rels = leftids; @@ -476,7 +536,8 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, (j->jointype != JOIN_INNER), &sub_qualscope, &sub_inners, - localEquiKeyList + localEquiKeyList, + &child_postponed_quals ); rightids = bms_add_members(rightids, sub_qualscope); *qualscope = bms_add_members(*qualscope, sub_qualscope); @@ -518,12 +579,38 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, ojscope = NULL; } + /* Try to process any quals postponed by children. If they need + * further postponement, add them to my output postponed_qual_list */ + foreach(qual, child_postponed_quals) + { + PostponedQual *pq = (PostponedQual *) lfirst(qual); + + if (bms_is_subset(pq->relids, *qualscope)) + { + distribute_qual_to_rels(root, pq->qual, + false, false, below_outer_join, + *qualscope, ojscope, nonnullable_rels, + ptrToLocalEquiKeyList, + NULL); + pfree(pq); + } + else + { + *postponed_qual_list = lappend(*postponed_qual_list, pq); + } + } + if (child_postponed_quals != NIL) + { + pfree(child_postponed_quals); + } + /* Process the qual clauses */ foreach(qual, (List *) j->quals) distribute_qual_to_rels(root, (Node *) lfirst(qual), false, false, below_outer_join, *qualscope, ojscope, nonnullable_rels, - ptrToLocalEquiKeyList); + ptrToLocalEquiKeyList, + postponed_qual_list); /* Now we can add the OuterJoinInfo to oj_info_list */ if (ojinfo) @@ -801,7 +888,8 @@ distribute_qual_to_rels(PlannerInfo *root, Node *clause, Relids qualscope, Relids ojscope, Relids outerjoin_nonnullable, - List **ptrToLocalEquiKeyList) + List **ptrToLocalEquiKeyList, + List **postponed_qual_list) { Relids relids; bool is_pushed_down; @@ -824,7 +912,22 @@ distribute_qual_to_rels(PlannerInfo *root, Node *clause, * Otherwise the parser messed up. */ if (!bms_is_subset(relids, qualscope)) - elog(ERROR, "JOIN qualification may not refer to other relations"); + { + if (postponed_qual_list == NULL) + { + elog(ERROR, "JOIN qualification may not refer to other relations"); + } + else + { + PostponedQual *pq = (PostponedQual *) palloc(sizeof(PostponedQual)); + + Assert(!is_deduced); + pq->qual = clause; + pq->relids = relids; + *postponed_qual_list = lappend(*postponed_qual_list, pq); + return; + } + } if (ojscope && !bms_is_subset(relids, ojscope)) elog(ERROR, "JOIN qualification may not refer to other relations"); @@ -1366,11 +1469,11 @@ process_implied_equality(PlannerInfo *root, */ distribute_qual_to_rels(root, (Node *) clause, true, true, false, relids, NULL, NULL, - NULL + NULL, /* NULL is okay for local equi list because * we are recording a global equivalence */ - ); + NULL); } /* diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 6e4cd4e170..e2c1707506 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -61,7 +61,6 @@ typedef struct reduce_outer_joins_state static void pull_up_fromlist_subqueries(PlannerInfo *root, List **inout_fromlist, - Node **inout_quals, bool below_outer_join); static Node *pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, @@ -174,8 +173,7 @@ pull_up_subqueries(PlannerInfo *root, Node *jtnode, FromExpr *f = (FromExpr *) jtnode; Assert(!append_rel_member); - pull_up_fromlist_subqueries(root, &f->fromlist, &f->quals, - below_outer_join); + pull_up_fromlist_subqueries(root, &f->fromlist, below_outer_join); } else if (IsA(jtnode, JoinExpr)) { @@ -226,7 +224,7 @@ pull_up_subqueries(PlannerInfo *root, Node *jtnode, * side of the JOIN (right side of LEFT JOIN). */ if (j->subqfromlist) - pull_up_fromlist_subqueries(root, &j->subqfromlist, &j->quals, + pull_up_fromlist_subqueries(root, &j->subqfromlist, below_outer_join || (j->jointype != JOIN_INNER)); } else @@ -243,7 +241,6 @@ pull_up_subqueries(PlannerInfo *root, Node *jtnode, static void pull_up_fromlist_subqueries(PlannerInfo *root, List **inout_fromlist, - Node **inout_quals, bool below_outer_join) { ListCell *l; @@ -254,30 +251,7 @@ pull_up_fromlist_subqueries(PlannerInfo *root, Node *newkid = pull_up_subqueries(root, oldkid, below_outer_join, false); - /* CDB: Collapse subquery FROM list into current FROM list, - * so correlated refs from subquery will be in the right scope. - * Otherwise deconstruct_jointree() complains about them. - */ - if (IsA(oldkid, RangeTblRef) && - IsA(newkid, FromExpr)) - { - FromExpr *fkid = (FromExpr *)newkid; - ListCell *lkid; - - /* Replace RangeTblRef with subquery FROM list. */ - foreach(lkid, fkid->fromlist) - { - if (lfirst(l) == oldkid) - lfirst(l) = lfirst(lkid); - else - l = lappend_cell(*inout_fromlist, l, lfirst(lkid)); - } - - /* Conjoin subquery WHERE clause with current search condition. */ - *inout_quals = make_and_qual(*inout_quals, fkid->quals); - } - else - lfirst(l) = newkid; + lfirst(l) = newkid; } } /* pull_up_fromlist_subqueries */ @@ -487,6 +461,18 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, ResolveNew((Node *) otherrte->joinaliasvars, varno, 0, rte, subtlist, CMD_SELECT, 0); + + else if (otherrte->rtekind == RTE_SUBQUERY && rte != otherrte) + { + otherrte->subquery = (Query *) + ResolveNew((Node *) otherrte->subquery, + varno, 1, rte, /* here the sublevels_up can only be 1, because if larger than 1, + then the sublink is multilevel correlated, and cannot be pulled + up to be a subquery range table; while on the other hand, we + cannot directly put a subquery which refer to other relations + of the same level after FROM. */ + subtlist, CMD_SELECT, 0); + } } /* @@ -633,17 +619,6 @@ is_simple_subquery(PlannerInfo *root, Query *subquery) if (contain_volatile_functions((Node *) subquery->targetList)) return false; - /* - * Don't pull up a subquery that coerces values to a different domain, since - * those may be used incorrectly before they are coerced by operators in - * the tree above - */ - List *corce_to_domain_list = extract_nodes(root->glob, (Node *) subquery, T_CoerceToDomain); - if (NIL != corce_to_domain_list) - { - return false; - } - /* * Hack: don't try to pull up a subquery with an empty jointree. * query_planner() will correctly generate a Result plan for a jointree diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index 4bbbbab427..c2d72ba664 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -246,7 +246,8 @@ extern void distribute_qual_to_rels(PlannerInfo *root, Node *clause, Relids qualscope, Relids ojscope, Relids outerjoin_nonnullable, - List **ptrToLocalEquiKeyList); + List **ptrToLocalEquiKeyList, + List **postponed_qual_list); /* diff --git a/src/test/regress/expected/gp_optimizer.out b/src/test/regress/expected/gp_optimizer.out index 79b82bf14e..23b1e5abdb 100644 --- a/src/test/regress/expected/gp_optimizer.out +++ b/src/test/regress/expected/gp_optimizer.out @@ -8602,6 +8602,82 @@ where c.cid = s.cid and s.date_sk = d.date_sk and (18 rows) reset optimizer_segments; +-- +-- apply parallelization for subplan MPP-24563 +-- +create table t1_mpp_24563 (id int, value int) distributed by (id); +insert into t1_mpp_24563 values (1, 3); +create table t2_mpp_24563 (id int, value int, seq int) distributed by (id); +insert into t2_mpp_24563 values (1, 7, 5); +set optimizer = off; +select row_number() over (order by seq asc) as id, foo.cnt +from +(select seq, (select count(*) from t1_mpp_24563 t1 where t1.id = t2.id) cnt from + t2_mpp_24563 t2 where value = 7) foo; + id | cnt +----+----- + 1 | 1 +(1 row) + +set optimizer = on; +select row_number() over (order by seq asc) as id, foo.cnt +from +(select seq, (select count(*) from t1_mpp_24563 t1 where t1.id = t2.id) cnt from + t2_mpp_24563 t2 where value = 7) foo; + id | cnt +----+----- + 1 | 1 +(1 row) + +drop table t1_mpp_24563; +drop table t2_mpp_24563; +-- +-- MPP-20470 update the flow of node after parallelizing subplan. +-- +CREATE TABLE t_mpp_20470 ( + col_date timestamp without time zone, + col_name character varying(6), + col_expiry date +) DISTRIBUTED BY (col_date) PARTITION BY RANGE(col_date) +( +START ('2013-05-10 00:00:00'::timestamp without time zone) END ('2013-05-11 + 00:00:00'::timestamp without time zone) WITH (tablename='t_mpp_20470_ptr1'), +START ('2013-05-24 00:00:00'::timestamp without time zone) END ('2013-05-25 + 00:00:00'::timestamp without time zone) WITH (tablename='t_mpp_20470_ptr2') +); +NOTICE: CREATE TABLE will create partition "t_mpp_20470_ptr1" for table "t_mpp_20470" +NOTICE: CREATE TABLE will create partition "t_mpp_20470_ptr2" for table "t_mpp_20470" +COPY t_mpp_20470 from STDIN delimiter '|' null ''; +create view v1_mpp_20470 as +SELECT +CASE + WHEN b.col_name::text = 'FUTCUR'::text + THEN ( SELECT count(a.col_expiry) AS count FROM t_mpp_20470 a WHERE + a.col_name::text = b.col_name::text)::text + ELSE 'Q2'::text END AS cc, 1 AS nn +FROM t_mpp_20470 b; +set optimizer = off; +SELECT cc, sum(nn) over() FROM v1_mpp_20470; + cc | sum +----+----- + Q2 | 4 + Q2 | 4 + 1 | 4 + Q2 | 4 +(4 rows) + +set optimizer = on; +SELECT cc, sum(nn) over() FROM v1_mpp_20470; + cc | sum +----+----- + Q2 | 4 + Q2 | 4 + 1 | 4 + Q2 | 4 +(4 rows) + +drop view v1_mpp_20470; +drop table t_mpp_20470; -- clean up drop schema orca cascade; NOTICE: drop cascades to table orca.bm_dyn_test_onepart_1_prt_part5 diff --git a/src/test/regress/expected/information_schema.out b/src/test/regress/expected/information_schema.out index e3a1634dc4..308462013d 100644 --- a/src/test/regress/expected/information_schema.out +++ b/src/test/regress/expected/information_schema.out @@ -64,4 +64,38 @@ where ordinal_position = 20; information_schema | routines | collation_catalog | 20 (13 rows) +-- MPP-25724 +select a.column_name +from information_schema.columns a +where a.table_name +in +(select b.table_name from information_schema.tables b where + a.column_name like 'b') and a.table_name = 'r'; + column_name +------------- + b +(1 row) + +select c.relname +from pg_class c +where c.relname +in +(select b.table_name from information_schema.tables b where + c.relname like 'r'); + relname +--------- + r +(1 row) + +select a.table_name +from information_schema.tables a +where a.table_name +in +(select b.relname from pg_class b where + a.table_name like 'r'); + table_name +------------ + r +(1 row) + drop table r; diff --git a/src/test/regress/expected/information_schema_optimizer.out b/src/test/regress/expected/information_schema_optimizer.out index e3a1634dc4..308462013d 100644 --- a/src/test/regress/expected/information_schema_optimizer.out +++ b/src/test/regress/expected/information_schema_optimizer.out @@ -64,4 +64,38 @@ where ordinal_position = 20; information_schema | routines | collation_catalog | 20 (13 rows) +-- MPP-25724 +select a.column_name +from information_schema.columns a +where a.table_name +in +(select b.table_name from information_schema.tables b where + a.column_name like 'b') and a.table_name = 'r'; + column_name +------------- + b +(1 row) + +select c.relname +from pg_class c +where c.relname +in +(select b.table_name from information_schema.tables b where + c.relname like 'r'); + relname +--------- + r +(1 row) + +select a.table_name +from information_schema.tables a +where a.table_name +in +(select b.relname from pg_class b where + a.table_name like 'r'); + table_name +------------ + r +(1 row) + drop table r; diff --git a/src/test/regress/sql/gp_optimizer.sql b/src/test/regress/sql/gp_optimizer.sql index 2f7e570e0e..9d02ae5061 100644 --- a/src/test/regress/sql/gp_optimizer.sql +++ b/src/test/regress/sql/gp_optimizer.sql @@ -713,5 +713,68 @@ from cust c, sales s, datedim d where c.cid = s.cid and s.date_sk = d.date_sk and ((d.year = 2001 and lower(s.type) = 't1' and plusone(d.moy) = 5) or (d.moy = 4 and upper(s.type) = 'T2')); reset optimizer_segments; + +-- +-- apply parallelization for subplan MPP-24563 +-- +create table t1_mpp_24563 (id int, value int) distributed by (id); +insert into t1_mpp_24563 values (1, 3); + +create table t2_mpp_24563 (id int, value int, seq int) distributed by (id); +insert into t2_mpp_24563 values (1, 7, 5); + +set optimizer = off; +select row_number() over (order by seq asc) as id, foo.cnt +from +(select seq, (select count(*) from t1_mpp_24563 t1 where t1.id = t2.id) cnt from + t2_mpp_24563 t2 where value = 7) foo; + +set optimizer = on; +select row_number() over (order by seq asc) as id, foo.cnt +from +(select seq, (select count(*) from t1_mpp_24563 t1 where t1.id = t2.id) cnt from + t2_mpp_24563 t2 where value = 7) foo; + +drop table t1_mpp_24563; +drop table t2_mpp_24563; + +-- +-- MPP-20470 update the flow of node after parallelizing subplan. +-- +CREATE TABLE t_mpp_20470 ( + col_date timestamp without time zone, + col_name character varying(6), + col_expiry date +) DISTRIBUTED BY (col_date) PARTITION BY RANGE(col_date) +( +START ('2013-05-10 00:00:00'::timestamp without time zone) END ('2013-05-11 + 00:00:00'::timestamp without time zone) WITH (tablename='t_mpp_20470_ptr1'), +START ('2013-05-24 00:00:00'::timestamp without time zone) END ('2013-05-25 + 00:00:00'::timestamp without time zone) WITH (tablename='t_mpp_20470_ptr2') +); + +COPY t_mpp_20470 from STDIN delimiter '|' null ''; +2013-05-10 00:00:00|OPTCUR|2013-05-29 +2013-05-10 04:35:20|OPTCUR|2013-05-29 +2013-05-24 03:10:30|FUTCUR|2014-04-28 +2013-05-24 05:32:34|OPTCUR|2013-05-29 +\. + +create view v1_mpp_20470 as +SELECT +CASE + WHEN b.col_name::text = 'FUTCUR'::text + THEN ( SELECT count(a.col_expiry) AS count FROM t_mpp_20470 a WHERE + a.col_name::text = b.col_name::text)::text + ELSE 'Q2'::text END AS cc, 1 AS nn +FROM t_mpp_20470 b; + +set optimizer = off; +SELECT cc, sum(nn) over() FROM v1_mpp_20470; +set optimizer = on; +SELECT cc, sum(nn) over() FROM v1_mpp_20470; + +drop view v1_mpp_20470; +drop table t_mpp_20470; -- clean up drop schema orca cascade; diff --git a/src/test/regress/sql/information_schema.sql b/src/test/regress/sql/information_schema.sql index ff8b06a951..ef2db0336c 100644 --- a/src/test/regress/sql/information_schema.sql +++ b/src/test/regress/sql/information_schema.sql @@ -32,4 +32,26 @@ select table_schema, table_name,column_name,ordinal_position from information_schema.columns where ordinal_position = 20; +-- MPP-25724 +select a.column_name +from information_schema.columns a +where a.table_name +in +(select b.table_name from information_schema.tables b where + a.column_name like 'b') and a.table_name = 'r'; + +select c.relname +from pg_class c +where c.relname +in +(select b.table_name from information_schema.tables b where + c.relname like 'r'); + +select a.table_name +from information_schema.tables a +where a.table_name +in +(select b.relname from pg_class b where + a.table_name like 'r'); + drop table r;