From daeb4ae8bc084d72d2c99fb7b52c2a391234cc84 Mon Sep 17 00:00:00 2001 From: Anoop Sharma Date: Mon, 26 Sep 2016 20:42:01 +0000 Subject: [PATCH 1/3] groupby rollup commit #1 --- core/sql/bin/SqlciErrors.txt | 1 + core/sql/comexe/ComTdbSortGrby.cpp | 3 +- core/sql/comexe/ComTdbSortGrby.h | 23 +- core/sql/executor/ex_globals.h | 12 + core/sql/executor/ex_sort_grby.cpp | 834 ++++++++++++++++++++++++- core/sql/executor/ex_sort_grby.h | 108 ++-- core/sql/exp/ExpPCodeClauseGen.cpp | 8 + core/sql/exp/exp_clause.cpp | 21 +- core/sql/exp/exp_clause_derived.h | 8 +- core/sql/exp/exp_comp.cpp | 108 ++-- core/sql/generator/GenExpGenerator.cpp | 1 - core/sql/generator/GenExplain.cpp | 14 +- core/sql/generator/GenItemExpr.cpp | 6 +- core/sql/generator/GenItemFunc.cpp | 2 + core/sql/generator/GenPreCode.cpp | 57 ++ core/sql/generator/GenRelGrby.cpp | 83 ++- core/sql/optimizer/BindRelExpr.cpp | 93 ++- core/sql/optimizer/ImplRule.cpp | 8 + core/sql/optimizer/ItemExpr.cpp | 17 +- core/sql/optimizer/ItemExpr.h | 9 +- core/sql/optimizer/ItemFunc.h | 10 +- core/sql/optimizer/ItemLog.h | 22 +- core/sql/optimizer/NormRelExpr.cpp | 22 +- core/sql/optimizer/RelCache.cpp | 4 + core/sql/optimizer/RelExpr.cpp | 10 +- core/sql/optimizer/RelGrby.h | 45 +- core/sql/optimizer/SynthType.cpp | 14 +- core/sql/optimizer/TransRule.cpp | 12 +- core/sql/parser/SqlParserAux.cpp | 11 +- core/sql/parser/sqlparser.y | 13 +- core/sql/regress/seabase/EXPECTED033 | 423 +++++++++++++ core/sql/regress/seabase/TEST033 | 123 ++++ 32 files changed, 1930 insertions(+), 195 deletions(-) create mode 100644 core/sql/regress/seabase/EXPECTED033 create mode 100644 core/sql/regress/seabase/TEST033 diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt index 3113d4a290..753bd0b62b 100644 --- a/core/sql/bin/SqlciErrors.txt +++ b/core/sql/bin/SqlciErrors.txt @@ -1372,6 +1372,7 @@ $1~String1 -------------------------------- 4381 42000 99999 BEGINNER MINOR LOGONLY Holdable cursor attribute is incompatible for the isolation level. 4382 42000 99999 BEGINNER MINOR LOGONLY Holdable cursor attribute cannot be set for a CALL statement. 4383 ZZZZZ 99999 BEGINNER MINOR DBADMIN A float value cannot be inserted into a numeric data type column $0~ColumnName, if the column is part of the partitioning key. +4384 ZZZZZ 99999 BEGINNER MINOR DBADMIN GROUP BY ROLLUP clause not allowed for this statement. Reason: $0~String0 4390 ZZZZZ 99999 BEGINNER MAJOR DBADMIN The OLAP History row size exceeds the limit of $0~Int0 bytes 4391 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Paramaters and outer references in the PARTITION BY or ORDER BY clause of a window function are not supported. 4400 ZZZZZ 99999 BEGINNER MAJOR LOGONLY Internal error diff --git a/core/sql/comexe/ComTdbSortGrby.cpp b/core/sql/comexe/ComTdbSortGrby.cpp index 0e8fb122ba..f35aa7efe9 100644 --- a/core/sql/comexe/ComTdbSortGrby.cpp +++ b/core/sql/comexe/ComTdbSortGrby.cpp @@ -83,7 +83,8 @@ ComTdbSortGrby::ComTdbSortGrby(ex_expr * aggr_expr, moveExpr_(move_expr), havingExpr_(having_expr), flags_(0), - tdbChild_(child_tdb) + tdbChild_(child_tdb), + numRollupGroups_(-1) { if (tolerateNonFatalError) setTolerateNonFatalError(TRUE); diff --git a/core/sql/comexe/ComTdbSortGrby.h b/core/sql/comexe/ComTdbSortGrby.h index fcf51f55be..05474131b8 100644 --- a/core/sql/comexe/ComTdbSortGrby.h +++ b/core/sql/comexe/ComTdbSortGrby.h @@ -46,8 +46,14 @@ class ComTdbSortGrby : public ComTdb { friend class ex_sort_grby_tcb; + friend class ex_sort_grby_rollup_tcb; friend class ex_sort_grby_private_state; + protected: + enum + { + IS_ROLLUP = 0x0001 + }; ComTdbPtr tdbChild_; // 00-07 ExExprPtr aggrExpr_; // 08-15 @@ -63,7 +69,9 @@ class ComTdbSortGrby : public ComTdb UInt16 flags_; // 46-47 - char fillersComTdbSortGrby_[40]; // 48-87 + Int16 numRollupGroups_; // 48-49 + + char fillersComTdbSortGrby_[38]; // 50-87 public: // Constructor @@ -90,6 +98,9 @@ NA_EIDPROC NA_EIDPROC ~ComTdbSortGrby(); + void setNumRollupGroups(Int16 v) { numRollupGroups_ = v; } + Int16 numRollupGroups() { return numRollupGroups_; } + // --------------------------------------------------------------------- // Redefine virtual functions required for Versioning. //---------------------------------------------------------------------- @@ -106,8 +117,6 @@ NA_EIDPROC ComTdb::populateImageVersionIDArray(); } - - NA_EIDPROC virtual short getClassSize() { return (short)sizeof(ComTdbSortGrby); } Long pack(void *); @@ -163,7 +172,11 @@ NA_EIDPROC else return NULL; } - + + NABoolean isRollup() {return ((flags_ & IS_ROLLUP) != 0);}; + void setIsRollup(NABoolean v) + {(v ? flags_ |= IS_ROLLUP : flags_ &= ~IS_ROLLUP);} + }; NA_EIDPROC @@ -175,7 +188,7 @@ inline ComTdb * ComTdbSortGrby::getChildTdb(){ Description : Return ComTdb* depending on the position argument. Position 0 means the left most child. Comments : - History : Yeogirl Yun 8/22/95 + History : Initial Revision. *****************************************************************************/ inline const ComTdb* ComTdbSortGrby::getChild(Int32 pos) const diff --git a/core/sql/executor/ex_globals.h b/core/sql/executor/ex_globals.h index f2c106b171..c9f20365e3 100644 --- a/core/sql/executor/ex_globals.h +++ b/core/sql/executor/ex_globals.h @@ -198,6 +198,9 @@ NA_EIDPROC Int64 &rowNum() { return rowNum_; } + void setRollupColumnNum(Int16 v) { rollupColumnNum_ = v; } + Int16 getRollupColumnNum() { return rollupColumnNum_; } + private: enum FlagsTypeEnum { @@ -277,6 +280,15 @@ NA_EIDPROC // Executor code does not access the contents. // void * lobGlob_; + // This value is set when grouping expression to compute rollup is + // evaluated. Caller (sort_grby_rollup_tcb) need to know the number + // of the grouping column that caused a comparison to fail. + // This value is set during comp clause. Caller resets it before and + // after call to expression evaluator. + // It is only a short duration setting. + // also see class ex_comp_clause in exp/exp_clause_derived.h on its usage. + Int16 rollupColumnNum_; + }; #endif diff --git a/core/sql/executor/ex_sort_grby.cpp b/core/sql/executor/ex_sort_grby.cpp index 7b6b9d14a6..2c7692263f 100644 --- a/core/sql/executor/ex_sort_grby.cpp +++ b/core/sql/executor/ex_sort_grby.cpp @@ -66,8 +66,13 @@ ex_tcb * ex_sort_grby_tdb::build(ex_globals * glob) child_tcb = tdbChild_->build(glob); - sort_grby_tcb = new(glob->getSpace()) - ex_sort_grby_tcb(*this, *child_tcb, glob); + if ((isRollup()) && + (numRollupGroups() > 0)) + sort_grby_tcb = new(glob->getSpace()) + ex_sort_grby_rollup_tcb(*this, *child_tcb, glob); + else + sort_grby_tcb = new(glob->getSpace()) + ex_sort_grby_tcb(*this, *child_tcb, glob); // add the sort_grby_tcb to the schedule, allow dynamic queue resizing sort_grby_tcb->registerSubtasks(); @@ -99,12 +104,10 @@ ex_sort_grby_tcb::ex_sort_grby_tcb(const ex_sort_grby_tdb & sort_grby_tdb, childTcb_ = &child_tcb; // Allocate the buffer pool -#pragma nowarn(1506) // warning elimination pool_ = new(space) ExSimpleSQLBuffer(sort_grby_tdb.numBuffers_, sort_grby_tdb.bufferSize_, recLen(), space); -#pragma warn(1506) // warning elimination // get the child queue pair qchild_ = child_tcb.getParentQueue(); @@ -159,6 +162,228 @@ void ex_sort_grby_tcb::freeResources() pool_ = 0; } +short ex_sort_grby_tcb::handleCancel(sort_grby_step &step, short &rc) +{ + // request was cancelled. Child was sent a cancel + // request. Ignore all up rows from child. + // Wait for EOF. + if (qchild_.up->isEmpty()) + { + // nothing returned from child. Get outta here. + rc = WORK_OK; + return -1; + } + + ex_queue_entry * centry = qchild_.up->getHeadEntry(); + + ex_assert(centry->upState.parentIndex == qparent_.down->getHeadIndex(), + "ex_sort_grby_tcb::work() child's reply out of sync"); + + ex_queue::up_status child_status = centry->upState.status; + switch(child_status) + { + case ex_queue::Q_OK_MMORE: + case ex_queue::Q_SQLERROR: + { + // just consume the child row + qchild_.up->removeHead(); + } + break; + + case ex_queue::Q_NO_DATA: + { + // return EOF to parent. + step = SORT_GRBY_DONE; + } + break; + + case ex_queue::Q_INVALID: + ex_assert(0,"ex_sort_grby_tcb::work() Invalid state returned by child"); + break; + + }; // end of switch on status of child queue + + return 0; +} + +short ex_sort_grby_tcb::handleError(sort_grby_step &step, short &rc) +{ + ex_queue_entry * pentry_down = qparent_.down->getHeadEntry(); + ex_sort_grby_private_state * pstate = + (ex_sort_grby_private_state*) pentry_down->pstate; + + qchild_.down->cancelRequestWithParentIndex( + qparent_.down->getHeadIndex()); + + // check if we've got room in the up queue + if (qparent_.up->isFull()) + { + rc = WORK_OK; // parent queue is full. Just return + return -1; + } + + ex_queue_entry *pentry_up = qparent_.up->getTailEntry(); + ex_queue_entry * centry = qchild_.up->getHeadEntry(); + + pentry_up->copyAtp(centry); + pentry_up->upState.parentIndex = + pentry_down->downState.parentIndex; + pentry_up->upState.downIndex = qparent_.down->getHeadIndex(); + pentry_up->upState.setMatchNo(pstate->matchCount_); + + if ((sort_grby_tdb().isNonFatalErrorTolerated()) && + (SORT_GRBY_LOCAL_ERROR == step)) + pentry_up->upState.status = ex_queue::Q_OK_MMORE; + else + pentry_up->upState.status = ex_queue::Q_SQLERROR; + + qparent_.up->insert(); + step = SORT_GRBY_CANCELLED; + + return 0; +} + +short ex_sort_grby_tcb::handleFinalize(sort_grby_step &step, + short &rc, + NABoolean noRelease) +{ + ex_queue_entry * pentry_down = qparent_.down->getHeadEntry(); + ex_sort_grby_private_state * pstate = + (ex_sort_grby_private_state*) pentry_down->pstate; + + // check if we've got room in the up queue + if (qparent_.up->isFull()) + { + rc = WORK_OK; // parent queue is full. Just return + return -1; + } + + ex_expr::exp_return_type evalRetCode = ex_expr::EXPR_OK; + if (havingExpr()) + { + evalRetCode = havingExpr()->eval(workAtp_, 0); + } + + if ((!havingExpr()) || + ((havingExpr()) && (evalRetCode == ex_expr::EXPR_TRUE))) + { + // if stats are to be collected, collect them. + if (getStatsEntry()) + { + getStatsEntry()->incActualRowsReturned(); + } + + // return to parent + ex_queue_entry * pentry_up = qparent_.up->getTailEntry(); + + pentry_up->copyAtp(workAtp_); + + pentry_up->upState.status = ex_queue::Q_OK_MMORE; + pentry_up->upState.parentIndex = pentry_down->downState.parentIndex; + pentry_up->upState.downIndex = qparent_.down->getHeadIndex(); + + pstate->matchCount_++; + pentry_up->upState.setMatchNo(pstate->matchCount_); + + // insert into parent up queue + qparent_.up->insert(); + } + else if (evalRetCode == ex_expr::EXPR_ERROR) + { + // The SORT_GRBY_LOCAL_ERROR state expects the + // diags area to be in the ATP of the head entry of + // the child's queue. It is currently in the workAtp_, + // so copy it to the childs head ATP. + // SORT_GRBY_LOCAL_ERROR will copy it from the + // child's queue entry to the parents up queue. + // + ex_queue_entry * centry = qchild_.up->getHeadEntry(); + centry->copyAtp(workAtp_); + step = SORT_GRBY_LOCAL_ERROR; + + return 0; + } + + // start a new group, if more rows in child's up queue + if (step == SORT_GRBY_FINALIZE_CANCEL) + step = SORT_GRBY_CANCELLED; + else + if (qchild_.up->getHeadEntry()->upState.status != + ex_queue::Q_OK_MMORE) + step = SORT_GRBY_DONE; + else + step = SORT_GRBY_NEW_GROUP; + + if (NOT noRelease) + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).release(); + + return 0; +} + +short ex_sort_grby_tcb::handleDone(sort_grby_step &step, short &rc, + NABoolean noAssert) +{ + ex_queue_entry * pentry_down = qparent_.down->getHeadEntry(); + ex_sort_grby_private_state * pstate = + (ex_sort_grby_private_state*) pentry_down->pstate; + + // check if we've got room in the up queue + if (qparent_.up->isFull()) + { + rc = WORK_OK; // parent queue is full. Just return + return -1; + } + + workAtp_->release(); + + ex_queue_entry * pentry_up = qparent_.up->getTailEntry(); + + pentry_up->upState.status = ex_queue::Q_NO_DATA; + pentry_up->upState.parentIndex = pentry_down->downState.parentIndex; + pentry_up->upState.downIndex = qparent_.down->getHeadIndex(); + pentry_up->upState.setMatchNo(pstate->matchCount_); + + if (step != SORT_GRBY_NEVER_STARTED) + { + ex_queue_entry * centry = qchild_.up->getHeadEntry(); + if (NOT noAssert) + { + ex_assert(centry->upState.status == ex_queue::Q_NO_DATA, + "ex_sort_grby_tcb::work() expecting Q_NO_DATA"); + ex_assert(centry->upState.parentIndex == qparent_.down->getHeadIndex(), + "ex_sort_grby_tcb::work() child's reply out of sync"); + } + qchild_.up->removeHead(); + } + + // remove the down entry + qparent_.down->removeHead(); + + // insert into parent up queue + qparent_.up->insert(); + + // re-initialize pstate + step = SORT_GRBY_EMPTY; + pstate->matchCount_ = 0; + workAtp_->release(); + + if (qparent_.down->isEmpty()) + { + rc = WORK_OK; + return -1; + } + + // If we haven't given to our child the new head + // index return and ask to be called again. + if (qparent_.down->getHeadIndex() == processedInputs_) + { + rc = WORK_CALL_AGAIN; + return -1; + } + + return 0; +} + //////////////////////////////////////////////////////////////////////////// // This is where the action is. //////////////////////////////////////////////////////////////////////////// @@ -661,6 +886,607 @@ short ex_sort_grby_tcb::work() } // while } +///////////////////////////////////////////////////////////// +// constructor ex_sort_grby_rollup_tcb +///////////////////////////////////////////////////////////// +ex_sort_grby_rollup_tcb::ex_sort_grby_rollup_tcb +(const ex_sort_grby_tdb & sgby_tdb, + const ex_tcb & child_tcb, // child queue pair + ex_globals * glob + ) : ex_sort_grby_tcb( sgby_tdb, child_tcb, glob), + rollupGroupAggrArr_(NULL), + step_(SORT_GRBY_EMPTY) +{ + rollupGroupAggrArr_ = new(glob->getDefaultHeap()) + char*[sort_grby_tdb().numRollupGroups()]; + + for (Int16 i = 0; i < sort_grby_tdb().numRollupGroups(); i++) + { + rollupGroupAggrArr_[i] = new(glob->getDefaultHeap()) char[recLen()]; + } +} + +short ex_sort_grby_rollup_tcb::rollupGrbyNull(Int16 groupNum) +{ + ExpTupleDesc * returnRowTD = + sort_grby_tdb().getCriDescUp()->getTupleDescriptor + (sort_grby_tdb().tuppIndex_); + + // returnRowTD has aggregate entries followed by group entries. + Int16 totalEntries = returnRowTD->numAttrs(); + Int16 numAggr = totalEntries - sort_grby_tdb().numRollupGroups(); + Int16 startEntry = numAggr + (groupNum > 0 ? groupNum : 0); + Int16 endEntry = totalEntries - 1; + char * grbyData = rollupGroupAggrArr_[groupNum]; + + // move nulls to rollup groups + for (Int16 i = startEntry; i <= endEntry; i++) + { + Attributes * attr = returnRowTD->getAttr(i); + if (attr->getNullFlag()) + { + *(short*)&grbyData[attr->getNullIndOffset()] = -1; + } + } + + return 0; +} + +short ex_sort_grby_rollup_tcb::rollupGrbyMove(ex_queue_entry * centry) +{ + // move initial group by value for the regular aggr + if (moveExpr()->eval(centry->getAtp(), workAtp_) == ex_expr::EXPR_ERROR) + { + return -1; + } + + // now move initial group by value for rollup aggrs + char * tempWorkDataPtr = + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).getDataPointer(); + + for (Int16 i = 0; i < sort_grby_tdb().numRollupGroups(); i++) + { + workAtp_->getTupp(sort_grby_tdb().tuppIndex_) + .setDataPointer(rollupGroupAggrArr_[i]); + + if (moveExpr()->eval(centry->getAtp(), workAtp_) == ex_expr::EXPR_ERROR) + { + workAtp_->getTupp(sort_grby_tdb().tuppIndex_) + .setDataPointer(tempWorkDataPtr); + + return -1; + } + } + + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).setDataPointer(tempWorkDataPtr); + + return 0; +} + +short ex_sort_grby_rollup_tcb::rollupAggrInit() +{ + if (! aggrExpr()) + return 0; + + // initialize the regular aggrs + if (((AggrExpr *)aggrExpr())->initializeAggr(workAtp_) == + ex_expr::EXPR_ERROR) + { + return -1; + } + + // now init rollup aggrs + char * tempWorkDataPtr = + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).getDataPointer(); + + for (Int16 i = startGroupNum_; i >= endGroupNum_; i--) + { + workAtp_->getTupp(sort_grby_tdb().tuppIndex_) + .setDataPointer(rollupGroupAggrArr_[i]); + + if (((AggrExpr *)aggrExpr())->initializeAggr(workAtp_) == + ex_expr::EXPR_ERROR) + { + workAtp_->getTupp(sort_grby_tdb().tuppIndex_) + .setDataPointer(tempWorkDataPtr); + return -1; + } + } + + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).setDataPointer(tempWorkDataPtr); + + return 0; +} + +short ex_sort_grby_rollup_tcb::rollupAggrEval(ex_queue_entry * centry) +{ + if (! aggrExpr()) + return 0; + + // evaluate the regular aggr + if (aggrExpr()->eval(centry->getAtp(), workAtp_) == ex_expr::EXPR_ERROR) + { + return -1; + } + + // now eval rollup aggrs + char * tempWorkDataPtr = + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).getDataPointer(); + + for (Int16 i = 0; i < sort_grby_tdb().numRollupGroups(); i++) + { + workAtp_->getTupp(sort_grby_tdb().tuppIndex_) + .setDataPointer(rollupGroupAggrArr_[i]); + + if (aggrExpr()->eval(centry->getAtp(), workAtp_) == ex_expr::EXPR_ERROR) + { + workAtp_->getTupp(sort_grby_tdb().tuppIndex_) + .setDataPointer(tempWorkDataPtr); + + return -1; + } + } + + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).setDataPointer(tempWorkDataPtr); + + return 0; +} + +//////////////////////////////////////////////////////////////////////////// +// work method for ex_sort_grby_rollup_tcb +// +// For: GROUP BY ROLLUP ( a, b, c) +// there will be 1 regular group and 3 rollup groups. +// Regular grouped result: (a,b,c) +// Rollup grouped result: (a,b), (a), (null) +// Rollup group number: 2 1 0 +// +// (null) is a special rollup group and represents the aggregated result +// across the whole table. +// +// sort_grby_tdb().numRollupGroups() will be set to 3 (M = 3). +// +// 3 aggregate rows will be allocated to hold results for rollup aggregates +// corresponding to the 3 rollup groups. +// temp row 0 is for aggr for group (null), 1 for (a), 2 for (a,b) +// +// In this work method, rollup grouping columns will be numbered from +// left to right and will be 1-based. +// Note: Last column ('c' in this case) is not used for rollup groups. +// +// For this example: +// Column: a b c +// Number(N): 1 2 3 +// +// Rollup column number is same as the rollup group number. +// (no rollup group for last column) +// +// +// Returning grouped results: +// +// A change in any of the grouping columns will first return the regular +// grouped result corresponding to (a,b,c) +// +// After that, rollup groups will be returned. +// Change in grouping col number N will cause rollup groups +// from (M-1) to N to be returned in decreasing order of group num. +// +// For ex: if 'b' changes, then rollup groups from 2(M-1) to 2(N) +// will be returned. In this case that will be group 2 (a,b) +// If 'a' changes, then rollup groups from 2(M-1) to 1(N) will be returned. +// In this case that will be groups 2(a,b) and 1(a) +// +// When EOD is reached, then all in-flight rollup groups are finalized +// and returned. +// +// After that, the final group 0 (null) is returned. +// +//////////////////////////////////////////////////////////////////////////// +short ex_sort_grby_rollup_tcb::work() +{ + short rc = 0; + + // if no parent request, return + if (qparent_.down->isEmpty()) + return WORK_OK; + + ex_queue_entry * pentry_down = qparent_.down->getHeadEntry(); + ex_sort_grby_private_state * pstate = + (ex_sort_grby_private_state*) pentry_down->pstate; + const ex_queue::down_request &request = pentry_down->downState.request; + + if (request == ex_queue::GET_NOMORE) + { + step_ = SORT_GRBY_NEVER_STARTED; + } + + while (1) // exit via return + { + // if we have already given to the parent all the rows needed cancel the + // parent's request. Also cancel it if the parent cancelled + if ((step_ != SORT_GRBY_CANCELLED) && + (step_ != SORT_GRBY_DONE) && + (step_ != SORT_GRBY_NEVER_STARTED) && + ((request == ex_queue::GET_NOMORE) || + ((request == ex_queue::GET_N) && + (pentry_down->downState.requestValue <= (Lng32)pstate->matchCount_)))) + { + qchild_.down->cancelRequestWithParentIndex(qparent_.down->getHeadIndex()); + step_ = SORT_GRBY_CANCELLED; + } + + switch (step_) + { + case SORT_GRBY_EMPTY: + { + ex_queue_entry * centry = qchild_.down->getTailEntry(); + + if ((sort_grby_tdb().firstNRows() >= 0) && + ((pentry_down->downState.request != ex_queue::GET_N) || + (pentry_down->downState.requestValue == sort_grby_tdb().firstNRows()))) + { + centry->downState.request = ex_queue::GET_N; + centry->downState.requestValue = sort_grby_tdb().firstNRows(); + } + else + { + centry->downState.request = ex_queue::GET_ALL; + centry->downState.requestValue = 11; + } + + centry->downState.parentIndex = processedInputs_; + + // set the child's input atp + centry->passAtp(pentry_down->getAtp()); + + qchild_.down->insert(); + + startGroupNum_ = sort_grby_tdb().numRollupGroups()-1; + endGroupNum_ = 0; + + allDone_ = FALSE; + step_ = SORT_GRBY_NEW_GROUP; + } + break; + + case SORT_GRBY_NEW_GROUP: + { + // Start of a new group. + // + // If a row is returned from child, + // create space for the grouped/aggregated row and move the + // grouping column value to it. + // + if (qchild_.up->isEmpty()) + { + // nothing returned from child. Get outta here. + return WORK_OK; + } + + ex_queue_entry * centry = qchild_.up->getHeadEntry(); + if (centry->upState.status == ex_queue::Q_SQLERROR) + { + step_ = SORT_GRBY_CHILD_ERROR; + break; + } + + if (centry->upState.status == ex_queue::Q_OK_MMORE) + { + // copy the down atp to the work atp for the current request + workAtp_->copyAtp(pentry_down->getAtp()); + + // get space to hold the grouped/aggregated row. + if (pool_->getFreeTuple(workAtp_->getTupp( + ((ex_sort_grby_tdb &)tdb).tuppIndex_))) + return WORK_POOL_BLOCKED; // couldn't allocate, try again. + + // move the group by value to this new row, if grouping + // is being done. + if (rollupGrbyMove(centry)) + { + step_ = SORT_GRBY_LOCAL_ERROR; + break; + } + + // initialize the aggregate + if (rollupAggrInit()) + { + step_ = SORT_GRBY_LOCAL_ERROR; + break; + } + + step_ = SORT_GRBY_STARTED; + break; + } + else + { + // no rows returned for a group by. + // Nothing to be returned(except, of course, Q_NO_DATA). + step_ = SORT_GRBY_DONE; + } + + } // start of a new group + break; + + case SORT_GRBY_STARTED: + { + if (qchild_.up->isEmpty()) + { + // nothing returned from child. Get outta here. + return WORK_OK; + } + + ex_queue_entry * centry = qchild_.up->getHeadEntry(); + ex_queue::up_status child_status = centry->upState.status; + switch(child_status) + { + case ex_queue::Q_OK_MMORE: + { + // row returned from child. Aggregate it. + // If the new child row belongs + // to this group, aggregate it. + + // rollupColumnNum will contain the column number + // which caused the change in group. + // This is a short duration global. Reset before the + // call to eval() method, set in eval() method, and then + // reset back afterwards. + getGlobals()->setRollupColumnNum(-1); + rollupColumnNum_ = -1; + ex_expr::exp_return_type grbyExprRetCode; + grbyExprRetCode = grbyExpr()->eval(centry->getAtp(), + workAtp_); + if (grbyExprRetCode == ex_expr::EXPR_FALSE) + { + // extract the position of groupby column that + // caused grby expr to fail. All rollup groupings + // greater than that will be finalized and returned. + rollupColumnNum_ = getGlobals()->getRollupColumnNum(); + } + + // reset global columnNum + getGlobals()->setRollupColumnNum(-1); + + if (grbyExprRetCode == ex_expr::EXPR_FALSE) + { + if (rollupColumnNum_ < 0) + { + step_ = SORT_GRBY_LOCAL_ERROR; + } + else + { + step_ = SORT_GRBY_FINALIZE; + } + } + else if (grbyExprRetCode == ex_expr::EXPR_ERROR) + { + step_ = SORT_GRBY_LOCAL_ERROR; + } + else + { + // aggregate the row. + if (rollupAggrEval(centry)) + { + step_ = SORT_GRBY_LOCAL_ERROR; + } + + // Here I check for only SORT_GRBY_FINALIZE_CANCEL + // because + // the logic is such that when we get here, the + // state cannot be SORT_GRBY_LOCAL_ERROR, + // SORT_GRBY_CHILD_ERROR or SORT_GRBY_FINALIZE. + else if (step_ != SORT_GRBY_FINALIZE_CANCEL) + qchild_.up->removeHead(); + } + } + break; + + case ex_queue::Q_NO_DATA: + { + // return current group to parent + step_ = SORT_GRBY_FINALIZE; + } + break; + + case ex_queue::Q_SQLERROR: + { + step_ = SORT_GRBY_CHILD_ERROR; + } + break; + + case ex_queue::Q_INVALID: + ex_assert(0,"ex_sort_grby_rollup_tcb::work() Invalid state returned by child"); + break; + + }; // end of switch on status of child queue + + } + break; + + case SORT_GRBY_CHILD_ERROR: + case SORT_GRBY_LOCAL_ERROR: + { + if (handleError(step_, rc)) + return rc; + + // next step set in handleError method + // step_ = SORT_GRBY_CANCELLED + + } // error + break; + + case SORT_GRBY_CANCELLED: + { + if (handleCancel(step_, rc)) + return rc; + + // next step set in handleCancel method + // step_ = SORT_GRBY_DONE + + } // request was cancelled + break; + + case SORT_GRBY_FINALIZE: + case SORT_GRBY_FINALIZE_CANCEL: + { + // if (handleFinalize(step_, rc, TRUE/*no workAtp release*/)) + if (handleFinalize(step_, rc)) + return rc; + + // next step set in handleFinalize method. + // Could be one of the following: + // step_ = SORT_GRBY_LOCAL_ERROR + // SORT_GRBY_CANCELLED + // SORT_GRBY_NEW_GROUP + // SORT_GRBY_DONE + + if (step_ == SORT_GRBY_NEW_GROUP) + { + // before starting on a new group, return rollup groups + step_ = SORT_GRBY_ROLLUP_GROUPS_INIT; + break; + } + else if (step_ == SORT_GRBY_DONE) + { + // return current rollup groups, final rollup group + // and then done. + allDone_ = TRUE; + + // EOD reached, mark rollupColumnNum as the first col. + // That will cause all in-flight groups to be returned. + rollupColumnNum_ = 1; + + step_ = SORT_GRBY_ROLLUP_GROUPS_INIT; + break; + } + + } // finalize + break; + + case SORT_GRBY_ROLLUP_GROUPS_INIT: + { + startGroupNum_ = sort_grby_tdb().numRollupGroups()-1; + endGroupNum_ = rollupColumnNum_; + currGroupNum_ = startGroupNum_; + + step_ = SORT_GRBY_ROLLUP_GROUP_START; + } + break; + + case SORT_GRBY_ROLLUP_GROUP_START: + { + // Groups are returned in reverse order. + // if done, start the next regular group. + if (currGroupNum_ < endGroupNum_) + { + if (allDone_) + step_ = SORT_GRBY_ROLLUP_FINAL_GROUP_START; + else + step_ = SORT_GRBY_NEW_GROUP; + break; + } + + // get space to hold the grouped/aggregated row. + if (pool_->getFreeTuple(workAtp_->getTupp( + sort_grby_tdb().tuppIndex_))) + return WORK_POOL_BLOCKED; // couldn't allocate, try again. + + step_ = SORT_GRBY_ROLLUP_GROUP; + } + break; + + case SORT_GRBY_ROLLUP_GROUP: + { + if (qparent_.up->isFull()) + return WORK_OK; // parent queue is full. Just return + + if (rollupGrbyNull(currGroupNum_)) + { + step_ = SORT_GRBY_LOCAL_ERROR; + break; + } + + // copy data from rollupArr to work atp so it could be returned. + char * currRollupDataPtr = rollupGroupAggrArr_[currGroupNum_]; + char * workDataPtr = + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).getDataPointer(); + str_cpy_all(workDataPtr, currRollupDataPtr, recLen()); + + if (handleFinalize(step_, rc)) + { + return rc; + } + + currGroupNum_--; + + step_ = SORT_GRBY_ROLLUP_GROUP_START; + } + break; + + case SORT_GRBY_ROLLUP_FINAL_GROUP_START: + { + // get space to hold the grouped/aggregated row. + if (pool_->getFreeTuple(workAtp_->getTupp( + sort_grby_tdb().tuppIndex_))) + return WORK_POOL_BLOCKED; // couldn't allocate, try again. + + step_ = SORT_GRBY_ROLLUP_FINAL_GROUP; + } + break; + + case SORT_GRBY_ROLLUP_FINAL_GROUP: + { + if (qparent_.up->isFull()) + { + return WORK_OK; // parent queue is full. Just return + } + + if (rollupGrbyNull(0)) + { + step_ = SORT_GRBY_LOCAL_ERROR; + break; + } + + // copy data from rollupArr to work atp so it could be returned. + char * currRollupDataPtr = rollupGroupAggrArr_[0]; + char * workDataPtr = + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).getDataPointer(); + str_cpy_all(workDataPtr, currRollupDataPtr, recLen()); + + if (handleFinalize(step_, rc)) + { + return rc; + } + + step_ = SORT_GRBY_DONE; + } + break; + + case SORT_GRBY_DONE: + case SORT_GRBY_NEVER_STARTED: + { + if (handleDone(step_, rc, TRUE/*no child entry assertion check*/)) + return rc; + + // step_: set in handleDone method + // value: SORT_GRBY_EMPTY + + return WORK_CALL_AGAIN; + } // done + break; + + default: + ex_assert(0, "Invalid step_ in ex_sort_grby_rollup_tcb::work()"); + break; + + } // switch step_ + + } // while +} + +/////////////////////////////////////////////////////////////////////////////// +// ex_sort_grby_tcb::allocatePstates +/////////////////////////////////////////////////////////////////////////////// ex_tcb_private_state * ex_sort_grby_tcb::allocatePstates( Lng32 &numElems, Lng32 &pstateLength) diff --git a/core/sql/executor/ex_sort_grby.h b/core/sql/executor/ex_sort_grby.h index 52fe8e9dc6..65c0bfbfe4 100644 --- a/core/sql/executor/ex_sort_grby.h +++ b/core/sql/executor/ex_sort_grby.h @@ -114,13 +114,14 @@ class ex_sort_grby_tdb : public ComTdbSortGrby // -// Task control block +// Task control block for ex_sort_grby_tcb // class ex_sort_grby_tcb : public ex_tcb { friend class ex_sort_grby_tdb; friend class ex_sort_grby_private_state; +protected: const ex_tcb * childTcb_; ex_queue_pair qparent_; @@ -155,73 +156,104 @@ NA_EIDPROC SORT_GRBY_CANCELLED, SORT_GRBY_LOCAL_ERROR, SORT_GRBY_CHILD_ERROR, - SORT_GRBY_NEVER_STARTED + SORT_GRBY_NEVER_STARTED, + + SORT_GRBY_ROLLUP_GROUPS_INIT, + SORT_GRBY_ROLLUP_GROUP_START, + SORT_GRBY_ROLLUP_GROUP, + SORT_GRBY_ROLLUP_FINAL_GROUP_START, + SORT_GRBY_ROLLUP_FINAL_GROUP }; -NA_EIDPROC void freeResources(); // free resources -NA_EIDPROC - short work(); // when scheduled to do work + virtual short work(); // when scheduled to do work -NA_EIDPROC - inline ex_sort_grby_tdb & sort_grby_tdb() const; + ex_sort_grby_tdb & sort_grby_tdb() const + {return (ex_sort_grby_tdb &) tdb;} -// return a pair of queue pointers to the parent node. Needed only during -// construction of nodes. -NA_EIDPROC + // return a pair of queue pointers to the parent node. Needed only during + // construction of nodes. + NA_EIDPROC ex_queue_pair getParentQueue() const -{ - return (qparent_); -} - - -NA_EIDPROC + { + return (qparent_); + } + virtual ex_tcb_private_state * allocatePstates( Lng32 &numElems, // inout, desired/actual elements Lng32 &pstateLength); // out, length of one element -NA_EIDPROC inline ex_expr * aggrExpr() const { return sort_grby_tdb().aggrExpr_; }; -NA_EIDPROC inline ex_expr * grbyExpr() const { return sort_grby_tdb().grbyExpr_; }; -NA_EIDPROC inline ex_expr * moveExpr() const { return sort_grby_tdb().moveExpr_; }; -NA_EIDPROC inline ex_expr * havingExpr() const { return sort_grby_tdb().havingExpr_; }; -NA_EIDPROC inline Lng32 recLen() {return sort_grby_tdb().recLen_;}; - NA_EIDPROC virtual Int32 numChildren() const { return 1; } - NA_EIDPROC virtual const ex_tcb* getChild(Int32 pos) const; + virtual Int32 numChildren() const { return 1; } + virtual const ex_tcb* getChild(Int32 pos) const + { + ex_assert((pos >= 0), ""); + if (pos == 0) + return childTcb_; + else + return NULL; + } + +protected: + short handleCancel(sort_grby_step &step, short &rc); + short handleError(sort_grby_step &step, short &rc); + short handleFinalize(sort_grby_step &step, short &rc, + NABoolean noRelease = FALSE); + short handleDone(sort_grby_step &step, short &rc, + NABoolean noAssert = FALSE); + }; -inline const ex_tcb* ex_sort_grby_tcb::getChild(Int32 pos) const + +///////////////////////////////////////////////////////////////////// +// Task control block for ex_sort_grby_rollup_tcb +///////////////////////////////////////////////////////////////////// +class ex_sort_grby_rollup_tcb : public ex_sort_grby_tcb { - ex_assert((pos >= 0), ""); - if (pos == 0) - return childTcb_; - else - return NULL; -} + friend class ex_sort_grby_tdb; + friend class ex_sort_grby_private_state; + +public: + // Constructor + ex_sort_grby_rollup_tcb(const ex_sort_grby_tdb & sort_grby_tdb, + const ex_tcb & child_tcb , // child queue pair + ex_globals * glob + ); + + virtual short work(); // when scheduled to do work +private: + short rollupAggrInit(); + short rollupGrbyNull(Int16 groupNum); + short rollupGrbyMove(ex_queue_entry * centry); + short rollupAggrEval(ex_queue_entry * centry); -/////////////////////////////////////////////////////////////////////////// -// -// Inline procedures -// -/////////////////////////////////////////////////////////////////////////// + char ** rollupGroupAggrArr_; -inline ex_sort_grby_tdb & ex_sort_grby_tcb::sort_grby_tdb() const -{ - return (ex_sort_grby_tdb &) tdb; + sort_grby_step step_; + + Int16 rollupColumnNum_; + Int16 startGroupNum_; + Int16 endGroupNum_; + Int16 currGroupNum_; + + NABoolean allDone_; }; + + /////////////////////////////////////////////////////////////////// class ex_sort_grby_private_state : public ex_tcb_private_state { friend class ex_sort_grby_tcb; + friend class ex_sort_grby_rollup_tcb; public: diff --git a/core/sql/exp/ExpPCodeClauseGen.cpp b/core/sql/exp/ExpPCodeClauseGen.cpp index 7ab9a35fe7..9bc6a182b6 100644 --- a/core/sql/exp/ExpPCodeClauseGen.cpp +++ b/core/sql/exp/ExpPCodeClauseGen.cpp @@ -804,6 +804,14 @@ ex_expr::exp_return_type ex_comp_clause::pCodeGenerate(Space *space, UInt32 f) { Attributes *op1 = attrs[1]; Attributes *op2 = attrs[2]; + // if this comp clause need to return the column num which caused the + // comparison to fail, then do not generate pcode. That functionality is + // not yet supported in pcode. + // This is used for rollup group computation which need to know the particular + // group that caused comparison to fail. + if (getRollupColumnNum() >= 0) + return ex_clause::pCodeGenerate(space, f); + // Generate the standard clause->eval PCode for cases that // are not handled with more fundamental PCode operations. // diff --git a/core/sql/exp/exp_clause.cpp b/core/sql/exp/exp_clause.cpp index 3ac18c112d..ca018abd21 100644 --- a/core/sql/exp/exp_clause.cpp +++ b/core/sql/exp/exp_clause.cpp @@ -1654,9 +1654,11 @@ ex_comp_clause::ex_comp_clause(OperatorTypeEnum oper_type, Space * space, ULng32 flags) : ex_clause (ex_clause::COMP_TYPE, oper_type, 3, attr, space), - flags_(0) + flags_(0), + rollupColumnNum_(-1) { - if(flags) setSpecialNulls(); + if(flags) + setSpecialNulls(); setInstruction(); } @@ -1942,8 +1944,19 @@ void ex_branch_clause::displayContents(Space * space, const char * /*displayStr* void ex_comp_clause::displayContents(Space * space, const char * /*displayStr*/, Int32 clauseNum, char * constsArea) { setInstruction(); - ex_clause::displayContents(space, "ex_comp_clause", clauseNum, constsArea, - flags_, + + char buf[100]; + str_sprintf(buf, " Clause #%d: ex_comp_clause", clauseNum); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short)); + + str_sprintf(buf, " ex_comp_clause::rollupColumnNum_ = %d", rollupColumnNum_); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short)); + + str_sprintf(buf, " ex_comp_clause::flags_ = %b", flags_); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short)); + + ex_clause::displayContents(space, (const char *)NULL, clauseNum, constsArea, + 0, ex_comp_clause::getInstruction(getInstrArrayIndex()), ex_comp_clause::getInstructionStr(getInstrArrayIndex())); diff --git a/core/sql/exp/exp_clause_derived.h b/core/sql/exp/exp_clause_derived.h index 5d9291d786..48b73c7ca0 100644 --- a/core/sql/exp/exp_clause_derived.h +++ b/core/sql/exp/exp_clause_derived.h @@ -1197,6 +1197,8 @@ class SQLEXP_LIB_FUNC ex_comp_clause : public ex_clause { Lng32 findIndexIntoInstrArray(CompInstruction ci); + void setRollupColumnNum(Int16 v) {rollupColumnNum_ = v;} + Int16 getRollupColumnNum() { return rollupColumnNum_; } // Null Semantics // @@ -1274,12 +1276,16 @@ class SQLEXP_LIB_FUNC ex_comp_clause : public ex_clause { Int32 filler0; // 00-03 Int16 flags_; //04-05 + + // see optimizer/ItemLog.h, class BiRelat + Int16 rollupColumnNum_; //06-07 + // --------------------------------------------------------------------- // Fillers for potential future extensions without changing class size. // When a new member is added, size of this filler should be reduced so // that the size of the object remains the same (and is modulo 8). // --------------------------------------------------------------------- - char fillers_[26]; // 06-31 + char fillers_[24]; // 08-31 NA_EIDPROC ex_expr::exp_return_type processResult(Int32 compare_code, Lng32* result, CollHeap *heap, ComDiagsArea** diagsArea); diff --git a/core/sql/exp/exp_comp.cpp b/core/sql/exp/exp_comp.cpp index 88ad46f5b7..d6bd42ae42 100644 --- a/core/sql/exp/exp_comp.cpp +++ b/core/sql/exp/exp_comp.cpp @@ -38,12 +38,12 @@ #include "Platform.h" - #include "exp_stdh.h" #include "exp_clause_derived.h" #include "exp_datetime.h" #include "unicode_char_set.h" #include "wstr.h" +#include "ex_globals.h" ex_expr::exp_return_type ex_comp_clause::processNulls(char *op_data[], CollHeap *heap, @@ -70,30 +70,24 @@ ex_expr::exp_return_type ex_comp_clause::processNulls(char *op_data[], case ITM_EQUAL: result = (left_is_null && right_is_null ? -1 : 0); break; - // LCOV_EXCL_START case ITM_NOT_EQUAL: result = (left_is_null && right_is_null ? 0 : -1); break; case ITM_GREATER: - // result = (left_is_null && (!right_is_null) ? -1 : 0); result = (right_is_null ? 0 : -1); break; case ITM_LESS: - // result = (left_is_null && (!right_is_null) ? 0 : -1); result = (left_is_null ? 0 : -1); break; case ITM_GREATER_EQ: result = (left_is_null ? -1 : 0); break; - // LCOV_EXCL_STOP case ITM_LESS_EQ: result = (right_is_null ? -1 : 0); break; - - } if (result) @@ -105,6 +99,12 @@ ex_expr::exp_return_type ex_comp_clause::processNulls(char *op_data[], else { *(Lng32 *)op_data[2 * MAX_OPERANDS] = 0; // result is FALSE + + if ((getRollupColumnNum() >= 0) && + (getExeGlobals())) + { + getExeGlobals()->setRollupColumnNum(getRollupColumnNum()); + } } return ex_expr::EXPR_NULL; } // one of the operands is a null value. @@ -127,50 +127,50 @@ ex_expr::exp_return_type ex_comp_clause::processNulls(char *op_data[], ex_expr::exp_return_type ex_comp_clause::processResult(Int32 compare_code, Lng32* result, - CollHeap *heap, - ComDiagsArea** diagsArea) + CollHeap *heap, + ComDiagsArea** diagsArea) { - *result = 0; - - switch (getOperType()) - { - case ITM_EQUAL: - if (compare_code == 0) - *result = 1; - break; - case ITM_NOT_EQUAL: - if (compare_code != 0) - *result = 1; - break; - - case ITM_LESS: - if (compare_code < 0) - *result = 1; - break; - - case ITM_LESS_EQ: - if (compare_code <= 0) - *result = 1; - break; - - case ITM_GREATER: - if (compare_code > 0) - *result = 1; - break; - - case ITM_GREATER_EQ: - if (compare_code >= 0) - *result = 1; - break; - - default: - // LCOV_EXCL_START - ExRaiseSqlError(heap, diagsArea, EXE_INTERNAL_ERROR); - return ex_expr::EXPR_ERROR; - // LCOV_EXCL_STOP - break; - } - return ex_expr::EXPR_OK; + *result = 0; + + switch (getOperType()) + { + case ITM_EQUAL: + if (compare_code == 0) + *result = 1; + break; + case ITM_NOT_EQUAL: + if (compare_code != 0) + *result = 1; + break; + + case ITM_LESS: + if (compare_code < 0) + *result = 1; + break; + + case ITM_LESS_EQ: + if (compare_code <= 0) + *result = 1; + break; + + case ITM_GREATER: + if (compare_code > 0) + *result = 1; + break; + + case ITM_GREATER_EQ: + if (compare_code >= 0) + *result = 1; + break; + + default: + // LCOV_EXCL_START + ExRaiseSqlError(heap, diagsArea, EXE_INTERNAL_ERROR); + return ex_expr::EXPR_ERROR; + // LCOV_EXCL_STOP + break; + } + return ex_expr::EXPR_OK; } ///////////////////////////////////////////////////////////////// @@ -1289,6 +1289,14 @@ ex_expr::exp_return_type ex_comp_clause::eval(char *op_data[], break; // LCOV_EXCL_STOP } + + if ((getRollupColumnNum() >= 0) && + (*(Lng32*)op_data[0] == 0) && + (getExeGlobals())) + { + getExeGlobals()->setRollupColumnNum(getRollupColumnNum()); + } + return retcode; } diff --git a/core/sql/generator/GenExpGenerator.cpp b/core/sql/generator/GenExpGenerator.cpp index d71eb053be..a7ff97f5f9 100644 --- a/core/sql/generator/GenExpGenerator.cpp +++ b/core/sql/generator/GenExpGenerator.cpp @@ -5600,7 +5600,6 @@ short ExpGenerator::genItemExpr(ItemExpr * item_expr, Attributes *** out_attr, } /* assign result attributes*/ - // attr[0] = map_table->getMapInfo(item_expr->getValueId())->getAttr(); attr[0] = map_info->getAttr(); if (gen_child) diff --git a/core/sql/generator/GenExplain.cpp b/core/sql/generator/GenExplain.cpp index b0ef8f9924..0a6fb9a431 100644 --- a/core/sql/generator/GenExplain.cpp +++ b/core/sql/generator/GenExplain.cpp @@ -1135,10 +1135,11 @@ GroupByAgg::addSpecificExplainInfo(ExplainTupleMaster *explainTuple, Generator *generator) { + NAString buffer; if (CmpCommon::getDefault(COMPRESSED_INTERNAL_FORMAT_EXPLAIN)==DF_ON && tdb->getNodeType() == ComTdb::ex_HASH_GRBY) { - NAString buffer = "variable_length_tuples: "; + buffer += "variable_length_tuples: "; if(((ComTdbHashGrby*)tdb)->useVariableLength()) { buffer += "yes "; @@ -1156,11 +1157,16 @@ GroupByAgg::addSpecificExplainInfo(ExplainTupleMaster *explainTuple, { buffer += "CIF: OFF "; } - - explainTuple->setDescription(buffer); } - return(explainTuple); + if (isRollup()) + { + buffer += "groupby_rollup: specified "; + } + + explainTuple->setDescription(buffer); + + return(explainTuple); } ExplainTuple* diff --git a/core/sql/generator/GenItemExpr.cpp b/core/sql/generator/GenItemExpr.cpp index 517272b67b..8c391ace51 100644 --- a/core/sql/generator/GenItemExpr.cpp +++ b/core/sql/generator/GenItemExpr.cpp @@ -573,6 +573,9 @@ short BiRelat::codeGen(Generator * generator) comp_clause->setCollationEncodeComp(getCollationEncodeComp()); + if (rollupColumnNum() >= 0) + comp_clause->setRollupColumnNum(rollupColumnNum()); + generator->getExpGenerator()->linkClause(this, comp_clause); } @@ -630,10 +633,11 @@ short Convert::codeGen(Generator * generator) ex_conv_clause * conv_clause = new(generator->getSpace()) ex_conv_clause(getOperatorType(), attr, generator->getSpace()); - conv_clause->setLastVOAoffset(lastVOAOffset_); + /* conv_clause->setLastVOAoffset(lastVOAOffset_); conv_clause->setLastNullIndicatorLength(lastNullIndicatorLength_); conv_clause->setLastVcIndicatorLength(lastVcIndicatorLength_); conv_clause->setAlignment(alignment_); + */ generator->getExpGenerator()->linkClause(this, conv_clause); diff --git a/core/sql/generator/GenItemFunc.cpp b/core/sql/generator/GenItemFunc.cpp index 1fb49d5813..bf08ca822b 100644 --- a/core/sql/generator/GenItemFunc.cpp +++ b/core/sql/generator/GenItemFunc.cpp @@ -1328,6 +1328,8 @@ short Cast::codeGen(Generator * generator) short CastType::codeGen(Generator * generator) { + if (makeNullable_) + return Cast::codeGen(generator); Attributes ** attr; diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp index a5af7fef88..3b417d2f3c 100644 --- a/core/sql/generator/GenPreCode.cpp +++ b/core/sql/generator/GenPreCode.cpp @@ -5745,6 +5745,55 @@ RelExpr * HashGroupBy::preCodeGen(Generator * generator, } +RelExpr *GroupByAgg::processGroupbyRollup(Generator * generator, + const ValueIdSet & externalInputs, + ValueIdSet &pulledNewInputs) +{ + // if not rollup or no groupby or num of grouping columns is 0, return. + if ((! isRollup()) || + (groupExprList().entries() == 0)) + return this; + + ValueId valId; + + // rollup groups need to be evaluated in the order that they were specified + // in the query. + // groupExprList() contains grouping items in user specified order. + // For each item in groupExpr() set, find its position in groupExprList(). + // Create a new list corresponding to the rearraned groupExpr() set. This list + // will be used during codeGen to generate rollup group expression. + for (valId = groupExpr().init(); + groupExpr().next(valId); + groupExpr().advance(valId)) + { + ItemExpr * itemExpr = valId.getItemExpr(); + + if (itemExpr->getOperatorType() != ITM_INDEXCOLUMN) + continue; + + NAColumn * nac = ((IndexColumn*)itemExpr)->getNAColumn(); + + NABoolean found = FALSE; + for (CollIndex ii = 0; + (NOT found && ii < groupExprList().entries()); ii++) + { + ValueId valId2 = groupExprList()[ii]; + ItemExpr * itemExpr2 = valId2.getItemExpr(); + if (itemExpr2->getOperatorType() != ITM_BASECOLUMN) + continue; + + NAColumn * nac2 = ((BaseColumn*)itemExpr2)->getNAColumn(); + found = (nac->getColName() == nac2->getColName()); + if (found) + { + groupExprList()[ii] = valId; + } + } // inner for + } // outer for + + return this; +} + RelExpr * GroupByAgg::preCodeGen(Generator * generator, const ValueIdSet & externalInputs, ValueIdSet &pulledNewInputs) @@ -5879,6 +5928,14 @@ RelExpr * GroupByAgg::preCodeGen(Generator * generator, } + if (isRollup()) + { + RelExpr * gbrExpr = processGroupbyRollup + (generator, externalInputs, pulledNewInputs); + if (! gbrExpr) + return NULL; + } + markAsPreCodeGenned(); // Done. diff --git a/core/sql/generator/GenRelGrby.cpp b/core/sql/generator/GenRelGrby.cpp index 07a35b9d25..9b549a03e6 100644 --- a/core/sql/generator/GenRelGrby.cpp +++ b/core/sql/generator/GenRelGrby.cpp @@ -67,6 +67,7 @@ short GroupByAgg::genAggrGrbyExpr(Generator * generator, ValueIdSet &aggregateExpr, ValueIdSet &groupExpr, + ValueIdList &groupExprList, ValueIdSet &selectionPred, Int32 workAtp, Int32 workAtpIndex, @@ -282,34 +283,64 @@ short GroupByAgg::genAggrGrbyExpr(Generator * generator, ValueIdList moveValIdList; ValueIdList gbyValIdList; ValueIdSet searchValIdSet; + + if (isRollup() && (NOT groupExprList.isEmpty())) { + for (CollIndex j = 0; j < groupExprList.entries(); j++) { + valId = groupExprList[j]; - if (NOT groupExpr.isEmpty()) { - for (valId = groupExpr.init(); - groupExpr.next(valId); - groupExpr.advance(valId), i++) { + ItemExpr * itemExpr = valId.getItemExpr(); + + // add this converted value to the map table. + ItemExpr * convNode = NULL; + convNode = new(generator->wHeap()) Convert (itemExpr); + // bind/type propagate the new node + convNode->bindNode(generator->getBindWA()); + + attrs[i++] = + (generator->addMapInfo(convNode->getValueId(), 0))->getAttr(); + moveValIdList.insert(convNode->getValueId()); + gbyValIdList.insert(valId); + + // add the search condition + BiRelat * biRelat = new(generator->wHeap()) + BiRelat(ITM_EQUAL, itemExpr, convNode); + biRelat->setSpecialNulls(-1); + biRelat->bindNode(generator->getBindWA()); + + biRelat->rollupColumnNum() = j+1; + + searchValIdSet.insert(biRelat->getValueId()); + } + } + else if (NOT groupExpr.isEmpty()) { + for (valId = groupExpr.init(); + groupExpr.next(valId); + groupExpr.advance(valId), i++) { + ItemExpr * itemExpr = valId.getItemExpr(); - + // add this converted value to the map table. Convert * convNode = new(generator->wHeap()) Convert (itemExpr); - + // bind/type propagate the new node convNode->bindNode(generator->getBindWA()); - + attrs[i] = (generator->addMapInfo(convNode->getValueId(), 0))->getAttr(); moveValIdList.insert(convNode->getValueId()); gbyValIdList.insert(valId); - + // add the search condition BiRelat * biRelat = new(generator->wHeap()) BiRelat(ITM_EQUAL, itemExpr, convNode); biRelat->setSpecialNulls(-1); biRelat->bindNode(generator->getBindWA()); + searchValIdSet.insert(biRelat->getValueId()); } } - + numAttrs = ( isAggrOneRow_ ? numAttrs + moveSet.entries() : numAttrs ); // Create the descriptor describing the aggr row and assign offset to attrs. @@ -449,18 +480,19 @@ short GroupByAgg::genAggrGrbyExpr(Generator * generator, // are input to this node. Input values already have location // assigned to them. if ((NOT getGroupAttr()->getCharacteristicInputs().contains(valId)) && - (valId.getItemExpr()->getOperatorType() != ITM_CONSTANT)) { - MapInfo * mapInfo = generator->addMapInfo(valId, 0); - Attributes * oldGroupColAttr = mapInfo->getAttr(); + (valId.getItemExpr()->getOperatorType() != ITM_CONSTANT)) + { + MapInfo * mapInfo = generator->addMapInfo(valId, 0); + Attributes * oldGroupColAttr = mapInfo->getAttr(); - oldGroupColAttr->copyLocationAttrs(newGroupColAttr); - oldGroupColAttr->setAtp(0); - oldGroupColAttr->setAtpIndex(returnedAtpIndex); + oldGroupColAttr->copyLocationAttrs(newGroupColAttr); + oldGroupColAttr->setAtp(0); + oldGroupColAttr->setAtpIndex(returnedAtpIndex); - // code has been generated for valId and a value is available. - // Mark it so. - mapInfo->codeGenerated(); - } + // code has been generated for valId and a value is available. + // Mark it so. + mapInfo->codeGenerated(); + } } } @@ -1744,7 +1776,8 @@ short GroupByAgg::codeGen(Generator * generator) { genAggrGrbyExpr(generator, aggregateExpr(), - groupExpr(), + groupExpr(), + groupExprList(), selectionPred(), 1, returnedDesc->noTuples() - 1, @@ -1784,12 +1817,18 @@ short GroupByAgg::codeGen(Generator * generator) { #pragma warn(1506) // warning elimination generator->initTdbFields(sortGrbyTdb); + if (isRollup()) + { + sortGrbyTdb->setIsRollup(TRUE); + + sortGrbyTdb->setNumRollupGroups(groupExprList().entries()); + } + if(!generator->explainDisabled()) { generator->setExplainTuple( addExplainInfo(sortGrbyTdb, childExplainTuple, 0, generator)); } - - + // set the new up cri desc. generator->setCriDesc(returnedDesc, Generator::UP); diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index 8e1d4d8795..7f479a22c5 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -4117,7 +4117,6 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase1(BindWA *bindWA) if (compExprTreeIsNull) return this; - if (currGroupByItemExpr->getOperatorType() == ITM_SEL_INDEX) { SelIndex * si = (SelIndex*)currGroupByItemExpr; @@ -4305,7 +4304,7 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase2(BindWA *bindWA) NABoolean specialMode = (CmpCommon::getDefault(MODE_SPECIAL_1) == DF_ON); - // make sure child of root is a groupby node.or a sequence node + // make sure child of root is a groupby node or a sequence node // whose child is a group by node if (child(0)->getOperatorType() != REL_GROUPBY && (child(0)->getOperatorType() != REL_SEQUENCE || @@ -4327,6 +4326,51 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase2(BindWA *bindWA) DCMPASSERT(grby != NULL); + if (grby->isRollup()) + { + if (grby->groupExpr().entries() != grby->groupExprList().entries()) + { + *CmpCommon::diags() << DgSqlCode(-4384) + << DgString0("Cannot have duplicate entries."); + + bindWA->setErrStatus(); + return NULL; + } + + for (ValueId valId = grby->aggregateExpr().init(); + grby->aggregateExpr().next(valId); + grby->aggregateExpr().advance(valId)) + { + ItemExpr * ae = valId.getItemExpr(); + + // right now, only support groupby rollup on min/max/sum/avg/count + if (NOT ((ae->getOperatorType() == ITM_MIN) || + (ae->getOperatorType() == ITM_MAX) || + (ae->getOperatorType() == ITM_SUM) || + (ae->getOperatorType() == ITM_AVG) || + (ae->getOperatorType() == ITM_COUNT) || + (ae->getOperatorType() == ITM_COUNT_NONULL))) + { + *CmpCommon::diags() << DgSqlCode(-4384) + << DgString0("Unsupported rollup aggregate function."); + + bindWA->setErrStatus(); + return NULL; + } + + // right now, only support groupby rollup on non-distinct aggrs + Aggregate * ag = (Aggregate*)ae; + if (ag->isDistinct()) + { + *CmpCommon::diags() << DgSqlCode(-4384) + << DgString0("Distinct rollup aggregates not supported."); + + bindWA->setErrStatus(); + return NULL; + } + } + } + ValueIdSet &groupExpr = grby->groupExpr(); // copy of groupExpr used to identify the changed // value ids @@ -4336,8 +4380,6 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase2(BindWA *bindWA) // these gets expanded at bind time, and so the select index have to // be offset with the expansion number since the sel_index number // reflects the select list at parse time. - - for (ValueId vid = groupExpr.init(); groupExpr.next(vid); groupExpr.advance(vid)) @@ -4351,8 +4393,11 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase2(BindWA *bindWA) si->setValueId(grpById); if (child(0)->getOperatorType() != REL_SEQUENCE) { - groupExprCpy.remove(vid); - groupExprCpy.insert(grpById); + groupExprCpy.remove(vid); + groupExprCpy.insert(grpById); + + CollIndex idx = grby->groupExprList().index(vid); + grby->groupExprList()[idx] = grpById; } else { //sequence @@ -4473,6 +4518,23 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase2(BindWA *bindWA) } // found Sel Index } + ValueId valId; + if (grby->isRollup()) + { + for (CollIndex i = 0; i < grby->groupExprList().entries(); i++) + { + valId = grby->groupExprList()[i]; + + if (NOT valId.getType().supportsSQLnull()) + { + *CmpCommon::diags() << DgSqlCode(-4384) + << DgString0("Grouped columns must be nullable."); + bindWA->setErrStatus(); + return NULL; + } + } + } + // recreate the groupExpr expression after updating the value ids grby->setGroupExpr (groupExprCpy); @@ -5042,13 +5104,8 @@ RelExpr *RelRoot::bindNode(BindWA *bindWA) NULL, NULL, NULL, child(0), PARSERHEAP()); le->setHandleInStringFormat(FALSE); setChild(0, le); - } - - } - - processRownum(bindWA); @@ -6893,15 +6950,17 @@ RelExpr *GroupByAgg::bindNode(BindWA *bindWA) ItemExpr *groupExprTree = removeGroupExprTree(); if (groupExprTree) { currScope->context()->inGroupByClause() = TRUE; - groupExprTree->convertToValueIdSet(groupExpr(), bindWA, ITM_ITEM_LIST); + groupExprTree->convertToValueIdList(groupExprList(), bindWA, ITM_ITEM_LIST); currScope->context()->inGroupByClause() = FALSE; if (bindWA->errStatus()) return this; - ValueIdList groupByList(groupExpr()); - for (CollIndex i = 0; i < groupByList.entries(); i++) + ValueIdSet vidSet(groupExprList()); + setGroupExpr(vidSet); + + for (CollIndex i = 0; i < groupExprList().entries(); i++) { - ValueId vid = groupByList[i]; + ValueId vid = groupExprList()[i]; vid.getItemExpr()->setIsGroupByExpr(TRUE); } @@ -6911,9 +6970,9 @@ RelExpr *GroupByAgg::bindNode(BindWA *bindWA) RETDesc * childRETDesc = child(0)->getRETDesc(); ItemExprList origSelectList(getParentRootSelectList(), bindWA->wHeap()); - for (CollIndex i = 0; i < groupByList.entries(); i++) + for (CollIndex i = 0; i < groupExprList().entries(); i++) { - ValueId vid = groupByList[i]; + ValueId vid = groupExprList()[i]; if((vid.getItemExpr()->getOperatorType() == ITM_SEL_INDEX)&& (((SelIndex*)(vid.getItemExpr()))->renamedColNameInGrbyClause())) { diff --git a/core/sql/optimizer/ImplRule.cpp b/core/sql/optimizer/ImplRule.cpp index 5421b68bb5..374f0becc0 100644 --- a/core/sql/optimizer/ImplRule.cpp +++ b/core/sql/optimizer/ImplRule.cpp @@ -1761,6 +1761,10 @@ NABoolean SortGroupByRule::topMatch (RelExpr *relExpr, if (grbyagg->groupExpr().isEmpty()) return FALSE; + // must use sortGroupBy for rollup aggregates + if (grbyagg->isRollup()) + return TRUE; + // Settings to limit Sort Group By application Lng32 sortGbySetting = CURRSTMT_OPTDEFAULTS->robustSortGroupBy(); if (context->getReqdPhysicalProperty()->getMustMatch() == NULL && @@ -2068,6 +2072,10 @@ NABoolean HashGroupByRule::topMatch (RelExpr *relExpr, context->getReqdPhysicalProperty())) return FALSE; + // groupby rollup is evaluated using SortGroupBy + if (grbyagg->isRollup()) + return FALSE; + return TRUE; } // HashGroupByRule::topMatch() diff --git a/core/sql/optimizer/ItemExpr.cpp b/core/sql/optimizer/ItemExpr.cpp index 15a36ad608..f48f26dd20 100644 --- a/core/sql/optimizer/ItemExpr.cpp +++ b/core/sql/optimizer/ItemExpr.cpp @@ -9721,6 +9721,8 @@ ItemExpr * BiRelat::copyTopNode(ItemExpr *derivedNode, CollHeap* outHeap) result->outerNullFilteringDetected_= outerNullFilteringDetected_; result->innerNullFilteringDetected_ = innerNullFilteringDetected_; + result->rollupColumnNum_ = rollupColumnNum_; + result->flags_ = flags_; return ItemExpr::copyTopNode(result, outHeap); @@ -12068,18 +12070,11 @@ Cast::Cast(ItemExpr *val1Ptr, const NAType *type, OperatorTypeEnum otype, { ValueId vid = val1Ptr ? val1Ptr->getValueId() : NULL_VALUE_ID; - if ((type->getFSDatatype() == 132) && - (vid != NULL_VALUE_ID) && - (vid.getType().getFSDatatype() == 136)) - { - Lng32 ij = 1; - } - checkForTruncation_ = FALSE; if (checkForTrunc) if (vid == NULL_VALUE_ID) checkForTruncation_ = TRUE; - else if ( type->getTypeQualifier() == NA_CHARACTER_TYPE && + else if ( type && type->getTypeQualifier() == NA_CHARACTER_TYPE && vid.getType().getTypeQualifier() == NA_CHARACTER_TYPE ) { if ( type->getNominalSize() < vid.getType().getNominalSize() ) @@ -12223,9 +12218,13 @@ ItemExpr * CastType::copyTopNode(ItemExpr *derivedNode, CollHeap* outHeap) ItemExpr *result; if (derivedNode == NULL) - result = new (outHeap) CastType(NULL, getType()->newCopy(outHeap)); + result = new (outHeap) + CastType(NULL, + (getType() ? getType()->newCopy(outHeap) : NULL)); else result = derivedNode; + + ((CastType*)result)->makeNullable_ = makeNullable_; return BuiltinFunction::copyTopNode(result, outHeap); } diff --git a/core/sql/optimizer/ItemExpr.h b/core/sql/optimizer/ItemExpr.h index c8364a1213..7add002c90 100644 --- a/core/sql/optimizer/ItemExpr.h +++ b/core/sql/optimizer/ItemExpr.h @@ -1210,6 +1210,10 @@ class ItemExpr : public ExprNode void setWasDefaultClause(NABoolean v) { (v ? flags_ |= WAS_DEFAULT_CLAUSE : flags_ &= ~WAS_DEFAULT_CLAUSE); } + NABoolean isGroupByRollup() const { return (flags_ & IS_GROUPBY_ROLLUP) != 0; } + void setIsGroupByRollup(NABoolean v) + { (v ? flags_ |= IS_GROUPBY_ROLLUP : flags_ &= ~IS_GROUPBY_ROLLUP); } + virtual QR::ExprElement getQRExprElem() const; virtual ItemExpr* removeRangeSpecItems(NormWA* normWA = NULL); @@ -1275,7 +1279,10 @@ class ItemExpr : public ExprNode // If set, this subtree was created while processing the DEFAULT // clause in DefaultSpecification::bindNode. - WAS_DEFAULT_CLAUSE = 0x0040 + WAS_DEFAULT_CLAUSE = 0x0040, + + // if set, the subtree rooted below was part of "groupby rollup" clause. + IS_GROUPBY_ROLLUP = 0x0080 }; // --------------------------------------------------------------------- diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h index 4acbf6c3ea..10086d354d 100644 --- a/core/sql/optimizer/ItemFunc.h +++ b/core/sql/optimizer/ItemFunc.h @@ -2546,7 +2546,13 @@ class CastType : public Cast { public: CastType(ItemExpr *val1Ptr, const NAType *type) - : Cast(val1Ptr, type, ITM_CAST_TYPE) + : Cast(val1Ptr, type, ITM_CAST_TYPE), + makeNullable_(FALSE) + {} + + CastType(ItemExpr *val1Ptr, NABoolean makeNullable) + : Cast(val1Ptr, NULL, ITM_CAST_TYPE), + makeNullable_(makeNullable) {} // copyTopNode method @@ -2558,6 +2564,8 @@ class CastType : public Cast virtual short codeGen(Generator*); +private: + NABoolean makeNullable_; }; // CastType class Narrow : public Cast diff --git a/core/sql/optimizer/ItemLog.h b/core/sql/optimizer/ItemLog.h index 99a28fb7c6..9e5240f986 100644 --- a/core/sql/optimizer/ItemLog.h +++ b/core/sql/optimizer/ItemLog.h @@ -346,6 +346,7 @@ class BiRelat : public ItemExpr isNotInPredTransform_(FALSE), outerNullFilteringDetected_ (FALSE), innerNullFilteringDetected_ (FALSE), + rollupColumnNum_(-1), flags_(0) { #ifndef NDEBUG @@ -452,18 +453,10 @@ class BiRelat : public ItemExpr return specialNulls_; } - //++MV - Irena: - // The getenv option is being used for testing purposes in MV void setSpecialNulls(NABoolean flag) { -#ifndef NDEBUG - if (NULL != getenv("FORCE_SPECIAL_NULLS")) { - specialNulls_ = TRUE; - } else -#endif - specialNulls_ = flag; + specialNulls_ = flag; } - //--MV - Irena: NABoolean & specialMultiValuePredicateTransformation() { return specialMultiValuePredicateTransformation_; } @@ -576,6 +569,8 @@ class BiRelat : public ItemExpr NABoolean getCollationEncodeComp() { return collationEncodeComp_;} virtual NABoolean hasEquivalentProperties(ItemExpr * other) { return TRUE;} + Int16 &rollupColumnNum() { return rollupColumnNum_; } + //Not In optimization methods // indicate that this birelat is a transformation of a NotIn. NABoolean getIsNotInPredTransform() const @@ -718,7 +713,6 @@ class BiRelat : public ItemExpr virtual QR::ExprElement getQRExprElem() const; - NABoolean collationEncodeComp_; // flag to indicate that this birelat is a transformation of a NotIn. @@ -730,7 +724,15 @@ class BiRelat : public ItemExpr NABoolean innerNullFilteringDetected_; + // Used for groupby rollup. + // Set in generator during groupby comparison expression generation. + // It indicates the position of grouping column that caused the group + // change during groupby computation. + // Used to return the rollup groups. + Int16 rollupColumnNum_; + UInt32 flags_; + }; // class BiRelat class KeyRangeCompare : public BiRelat diff --git a/core/sql/optimizer/NormRelExpr.cpp b/core/sql/optimizer/NormRelExpr.cpp index 4cdf1545d6..2c623503fc 100644 --- a/core/sql/optimizer/NormRelExpr.cpp +++ b/core/sql/optimizer/NormRelExpr.cpp @@ -5344,10 +5344,24 @@ RelExpr * GroupByAgg::normalizeNode(NormWA & normWARef) // Check if any of the HAVING clause predicates can be pushed down // (only when a Group By list is given). // --------------------------------------------------------------------- - pushdownCoveredExpr(getGroupAttr()->getCharacteristicOutputs(), - getGroupAttr()->getCharacteristicInputs(), - selectionPred() - ); + + // if this is a rollup groupby, then do not pushdown having pred to + // child node. If pushdown is done, then it might eliminate rows that + // are generated during rollup groupby processing. + // For ex: + // select a from t group by rollup(a) having a is not null; + // If 'having' pred is pushdown to scan node, then it will not return + // any rows from scan and will elimite processing of rollup groups + // that are returned by groupby. + // Maybe later we can optimize so this pushdown is done if possible, + // for ex, if there are no 'is null/not null' preds on grouping cols. + if (NOT isRollup()) + { + pushdownCoveredExpr(getGroupAttr()->getCharacteristicOutputs(), + getGroupAttr()->getCharacteristicInputs(), + selectionPred() + ); + } NABoolean needsNewVEGRegion = FALSE; diff --git a/core/sql/optimizer/RelCache.cpp b/core/sql/optimizer/RelCache.cpp index 77d3aad2d4..8c1440a09b 100644 --- a/core/sql/optimizer/RelCache.cpp +++ b/core/sql/optimizer/RelCache.cpp @@ -418,6 +418,10 @@ void GroupByAgg::generateCacheKey(CacheWA &cwa) const groupExpr_.rebuildExprTree(ITM_ITEM_LIST); if (grpExpr) { cwa += " gBy:"; + + if (isRollup()) + cwa += " roll:"; + grpExpr->generateCacheKey(cwa); } } diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp index b39bdda5ba..1785bfc18b 100644 --- a/core/sql/optimizer/RelExpr.cpp +++ b/core/sql/optimizer/RelExpr.cpp @@ -7523,6 +7523,7 @@ RelExpr * GroupByAgg::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) result->aggregateExprTree_ = aggregateExprTree_->copyTree(outHeap); result->groupExpr_ = groupExpr_; + result->groupExprList_ = groupExprList_; result->aggregateExpr_ = aggregateExpr_; result->formEnum_ = formEnum_; result->gbAggPushedBelowTSJ_ = gbAggPushedBelowTSJ_; @@ -7536,6 +7537,8 @@ RelExpr * GroupByAgg::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) result->selIndexInHaving_ = selIndexInHaving_; result->aggrExprsToBeDeleted_ = aggrExprsToBeDeleted_; + result->isRollup_ = isRollup_; + return RelExpr::copyTopNode(result, outHeap); } @@ -7672,6 +7675,8 @@ void GroupByAgg::addLocalExpr(LIST(ExprNode *) &xlist, { if (groupExpr_.isEmpty()) xlist.insert(groupExprTree_); + else if (isRollup() && (NOT groupExprList_.isEmpty())) + xlist.insert(groupExprList_.rebuildExprTree(ITM_ITEM_LIST)); else xlist.insert(groupExpr_.rebuildExprTree(ITM_ITEM_LIST)); llist.insert("grouping_columns"); @@ -7827,7 +7832,10 @@ NABoolean SortGroupBy::isPhysical() const {return TRUE;} const NAString SortGroupBy::getText() const { - return "sort_" + GroupByAgg::getText(); + if (isRollup()) + return "sort_" + GroupByAgg::getText() + "_rollup"; + else + return "sort_" + GroupByAgg::getText(); } RelExpr * SortGroupBy::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) diff --git a/core/sql/optimizer/RelGrby.h b/core/sql/optimizer/RelGrby.h index 669b2df5c4..9a644ae415 100644 --- a/core/sql/optimizer/RelGrby.h +++ b/core/sql/optimizer/RelGrby.h @@ -95,10 +95,10 @@ class GroupByAgg : public RelExpr // constructor // warning elimination (removed "inline") GroupByAgg(RelExpr *child, - OperatorTypeEnum otype = REL_GROUPBY, - ItemExpr *groupExpr = NULL, - ItemExpr *aggregateExpr = NULL, - CollHeap *oHeap = CmpCommon::statementHeap()) + OperatorTypeEnum otype = REL_GROUPBY, + ItemExpr *groupExpr = NULL, + ItemExpr *aggregateExpr = NULL, + CollHeap *oHeap = CmpCommon::statementHeap()) : RelExpr(otype, child, NULL, oHeap), groupExprTree_(groupExpr), aggregateExprTree_(aggregateExpr), @@ -111,7 +111,8 @@ class GroupByAgg : public RelExpr parentRootSelectList_(NULL), isMarkedForElimination_(FALSE), aggDistElimRuleCreates_(FALSE), - groupByOnJoinRuleCreates_(FALSE) + groupByOnJoinRuleCreates_(FALSE), + isRollup_(FALSE) {} // constructor @@ -128,7 +129,8 @@ class GroupByAgg : public RelExpr parentRootSelectList_(NULL), isMarkedForElimination_(FALSE), aggDistElimRuleCreates_(FALSE), - groupByOnJoinRuleCreates_(FALSE) + groupByOnJoinRuleCreates_(FALSE), + isRollup_(FALSE) {} // virtual destructor @@ -150,6 +152,13 @@ class GroupByAgg : public RelExpr inline void setGroupExpr(ValueIdSet &expr) { groupExpr_ = expr;} inline void addGroupExpr(ValueIdSet &expr) { groupExpr_ += expr;} + ValueIdList & groupExprList() { return groupExprList_; } + const ValueIdList & groupExprList() const { return groupExprList_; } + void setGroupExprList(ValueIdList &expr) { groupExprList_ = expr;} + + inline ValueIdList & rollupAggrExpr() { return rollupAggrExpr_; } + inline const ValueIdList & rollupAggrExpr() const { return rollupAggrExpr_; } + // return a (short-lived) read/write reference to the item expressions inline ValueIdSet & leftUniqueExpr() { return leftUniqueExpr_; } inline const ValueIdSet & leftUniqueExpr() const { return leftUniqueExpr_; } @@ -399,7 +408,9 @@ class GroupByAgg : public RelExpr // SortGroupBy::codeGen. // Defined in file GenRelGrby.C short genAggrGrbyExpr(Generator * generator, - ValueIdSet &aggregateExpr, ValueIdSet &groupExpr, + ValueIdSet &aggregateExpr, + ValueIdSet &groupExpr, + ValueIdList &groupExprList, ValueIdSet &selectionPred, Int32 workAtp, Int32 workAtpIndex, short returnedAtpIndex, @@ -516,10 +527,13 @@ class GroupByAgg : public RelExpr ValueIdSet &pulledPredicates, // return the pulled-up preds ValueIdMap *optionalMap); // optional map to rewrite preds - /*ExpTupleDesc::TupleDataFormat determineInternalFormat( const ValueIdList & valIdList, - RelExpr * relExpr, - NABoolean & resizeCifRecord, - Generator * generator);*/ + RelExpr *processGroupbyRollup(Generator * generator, + const ValueIdSet & externalInputs, + ValueIdSet &pulledNewInputs); + + void setIsRollup(NABoolean v) { isRollup_ = v; } + NABoolean isRollup() { return isRollup_; } + const NABoolean isRollup() const { return isRollup_; } ////////////////////////////////////////////////////// @@ -545,6 +559,13 @@ class GroupByAgg : public RelExpr // --------------------------------------------------------------------- ItemExpr * groupExprTree_; ValueIdSet groupExpr_; + + // -------------------------------------- + // used for processing groupby rollup + // -------------------------------------- + ValueIdList groupExprList_; + ValueIdList rollupAggrExpr_; + // --------------------------------------------------------------------- // The expression specifying the aggregates to be generated // --------------------------------------------------------------------- @@ -618,6 +639,8 @@ class GroupByAgg : public RelExpr NABoolean isMarkedForElimination_; ValueIdSet aggrExprsToBeDeleted_; + + NABoolean isRollup_; }; class SortGroupBy : public GroupByAgg diff --git a/core/sql/optimizer/SynthType.cpp b/core/sql/optimizer/SynthType.cpp index 2b9a1322d9..477e2ccd09 100644 --- a/core/sql/optimizer/SynthType.cpp +++ b/core/sql/optimizer/SynthType.cpp @@ -2515,9 +2515,17 @@ const NAType *CastConvert::synthesizeType() const NAType *CastType::synthesizeType() { - // NABuiltInTypeEnum qual = child(0)->getValueId().getType().getTypeQualifier(); - // if (qual != NA_CHARACTER_TYPE) - // return NULL; // source must be a character type + if (getType()) + return getType(); + + ValueId childVid = child(0)->getValueId(); + NAType *newType = childVid.getType().newCopy(HEAP); + if (makeNullable_ && (NOT newType->supportsSQLnull())) + { + newType->setNullable(TRUE); + } + changeType(newType); + return getType(); } diff --git a/core/sql/optimizer/TransRule.cpp b/core/sql/optimizer/TransRule.cpp index 356e44a3f4..411a8b7215 100644 --- a/core/sql/optimizer/TransRule.cpp +++ b/core/sql/optimizer/TransRule.cpp @@ -3910,6 +3910,10 @@ NABoolean GroupByEliminationRule::topMatch(RelExpr * expr, if (grby->groupExpr().isEmpty()) return FALSE; + // do not eliminate group by if rollup is being done + if (grby->isRollup()) + return FALSE; + return (grby->child(0).getGroupAttr()->isUnique(grby->groupExpr())); } @@ -4994,7 +4998,13 @@ NABoolean GroupByOnJoinRule::topMatch (RelExpr * expr, // Do not split the group by if it can be eliminated if (NOT bef->groupExpr().isEmpty()) - return NOT (bef->child(0).getGroupAttr()->isUnique(bef->groupExpr())); + { + // do not eliminate group by if rollup is being done + if (bef->isRollup()) + return FALSE; + + return NOT (bef->child(0).getGroupAttr()->isUnique(bef->groupExpr())); + } // the functional dependencies shown below won't hold if this is an // aggregate (ok, there are some sick examples where they do, but diff --git a/core/sql/parser/SqlParserAux.cpp b/core/sql/parser/SqlParserAux.cpp index 0a86730b5d..dfd625cee9 100644 --- a/core/sql/parser/SqlParserAux.cpp +++ b/core/sql/parser/SqlParserAux.cpp @@ -2684,14 +2684,17 @@ RelExpr * getTableExpressionRelExpr( *SqlParser_Diags << DgSqlCode(-4363); return NULL; } - if((groupByClause || havingClause) && (NOT groupByClauseProcessed)) - { - childPtr = new (PARSERHEAP()) + if((groupByClause || havingClause) && (NOT groupByClauseProcessed)) + { + childPtr = new (PARSERHEAP()) GroupByAgg(childPtr, REL_GROUPBY, groupByClause); // add having clause as a selection pred childPtr->addSelPredTree(havingClause); + + if (groupByClause) + ((GroupByAgg*)childPtr)->setIsRollup(groupByClause->isGroupByRollup()); } // sequence node goes right below rel root @@ -2701,7 +2704,7 @@ RelExpr * getTableExpressionRelExpr( seqNode->setHasOlapFunctions(hasOlapFunctions); childPtr = seqNode; } - } + } // !hasTDFunctions else { if (!seqByClause) diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y index c6213e002d..ae288b0f13 100755 --- a/core/sql/parser/sqlparser.y +++ b/core/sql/parser/sqlparser.y @@ -2083,9 +2083,7 @@ static void enableMakeQuotedStringISO88591Mechanism() %type join_condition %type where_clause %type multi_commit_size -//%type group_by_clause %type group_by_clause_non_empty -//%type having_clause %type having_clause_non_empty %type qualify_clause %type sort_by_key @@ -10686,6 +10684,12 @@ cast_specification : TOK_CAST '(' value_expression TOK_AS Set_Cast_Global_Fals CastType($3, $5); empty_charlen_specifier_allowed = FALSE; } + | TOK_CAST '(' value_expression TOK_AS TOK_NULLABLE ')' + { + $$ = new (PARSERHEAP()) + CastType($3, TRUE); + empty_charlen_specifier_allowed = FALSE; + } /* proc_arg_data_type are the types that are generated by the preprocessor as the arguments of a PROCEDURE statement. The tandem_float_type are the @@ -13167,6 +13171,11 @@ where_clause : { $$ = NULL; } group_by_clause_non_empty : TOK_GROUP TOK_BY value_expression_list { $$ = $3; } + | TOK_GROUP TOK_BY TOK_ROLLUP '(' value_expression_list ')' + { + $5->setIsGroupByRollup(TRUE); + $$ = $5; + } /* type item */ having_clause_non_empty : TOK_HAVING search_condition diff --git a/core/sql/regress/seabase/EXPECTED033 b/core/sql/regress/seabase/EXPECTED033 new file mode 100644 index 0000000000..f1ec8e07cc --- /dev/null +++ b/core/sql/regress/seabase/EXPECTED033 @@ -0,0 +1,423 @@ +>> +>>drop table if exists t033t1 cascade; + +--- SQL operation complete. +>>drop table if exists t033t2 cascade; + +--- SQL operation complete. +>>drop table if exists t033t3 cascade; + +--- SQL operation complete. +>> +>>create table t033t1 (a int, b int, c int, d int not null); + +--- SQL operation complete. +>> +>>insert into t033t1 values (1,2,3,4),(2,3,4,5),(3,3,3,3),(3,3,4,5),(1,3,3,3), ++> (null,null,null,6); + +--- 6 row(s) inserted. +>> +>>select * from t033t1 order by 1,2,3,4; + +A B C D +----------- ----------- ----------- ----------- + + 1 2 3 4 + 1 3 3 3 + 2 3 4 5 + 3 3 3 3 + 3 3 4 5 + ? ? ? 6 + +--- 6 row(s) selected. +>> +>>explain options 'f' select a,b,c,sum(d) from t033t1 ++> group by rollup(a,b,c) order by 1,2,3; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +3 . 4 root 8.00E+000 +2 . 3 sort_groupby_rollup 8.00E+000 +1 . 2 sort 1.00E+002 +. . 1 trafodion_scan T033T1 1.00E+002 + +--- SQL operation complete. +>>select a,b,c,sum(d) from t033t1 group by rollup(a,b,c) order by 1,2,3; + +A B C (EXPR) +----------- ----------- ----------- -------------------- + + 1 2 3 4 + 1 2 ? 4 + 1 3 3 3 + 1 3 ? 3 + 1 ? ? 7 + 2 3 4 5 + 2 3 ? 5 + 2 ? ? 5 + 3 3 3 3 + 3 3 4 5 + 3 3 ? 8 + 3 ? ? 8 + ? ? ? 6 + ? ? ? 6 + ? ? ? 6 + ? ? ? 26 + +--- 16 row(s) selected. +>> +>>select * from ( ++>select a,b,c,sum(d) from t033t1 group by (a,b,c) ++>union all ++>select a,b,cast(null as int),sum(d) from t033t1 group by (a,b) ++>union all ++>select a,cast(null as int),cast(null as int),sum(d) from t033t1 group by (a) ++>union all ++>select cast(null as int),cast(null as int),cast(null as int),sum(d) from t033t1 ++>) x(a,b,c,d) ++>order by a,b,c; + +A B C D +----------- ----------- ----------- -------------------- + + 1 2 3 4 + 1 2 ? 4 + 1 3 3 3 + 1 3 ? 3 + 1 ? ? 7 + 2 3 4 5 + 2 3 ? 5 + 2 ? ? 5 + 3 3 3 3 + 3 3 4 5 + 3 3 ? 8 + 3 ? ? 8 + ? ? ? 6 + ? ? ? 6 + ? ? ? 6 + ? ? ? 26 + +--- 16 row(s) selected. +>> +>>select b,c,a,sum(d) from t033t1 group by rollup(b,c,a) order by 1,2,3; + +B C A (EXPR) +----------- ----------- ----------- -------------------- + + 2 3 1 4 + 2 3 ? 4 + 2 ? ? 4 + 3 3 1 3 + 3 3 3 3 + 3 3 ? 6 + 3 4 2 5 + 3 4 3 5 + 3 4 ? 10 + 3 ? ? 16 + ? ? ? 6 + ? ? ? 6 + ? ? ? 6 + ? ? ? 26 + +--- 14 row(s) selected. +>> +>>select * from ( ++>select b,c,a,sum(d) from t033t1 group by (b,c,a) ++>union all ++>select b,c,cast(null as int),sum(d) from t033t1 group by (b,c) ++>union all ++>select b,cast(null as int),cast(null as int),sum(d) from t033t1 group by (b) ++>union all ++>select cast(null as int),cast(null as int),cast(null as int),sum(d) from t033t1 ++>) x(a,b,c,d) ++>order by a,b,c; + +A B C D +----------- ----------- ----------- -------------------- + + 2 3 1 4 + 2 3 ? 4 + 2 ? ? 4 + 3 3 1 3 + 3 3 3 3 + 3 3 ? 6 + 3 4 2 5 + 3 4 3 5 + 3 4 ? 10 + 3 ? ? 16 + ? ? ? 6 + ? ? ? 6 + ? ? ? 6 + ? ? ? 26 + +--- 14 row(s) selected. +>> +>>select a,b,c from t033t1 group by rollup (a,b,c) order by 1,2,3; + +A B C +----------- ----------- ----------- + + 1 2 3 + 1 2 ? + 1 3 3 + 1 3 ? + 1 ? ? + 2 3 4 + 2 3 ? + 2 ? ? + 3 3 3 + 3 3 4 + 3 3 ? + 3 ? ? + ? ? ? + ? ? ? + ? ? ? + ? ? ? + +--- 16 row(s) selected. +>> +>>select a, count(distinct b) from t033t1 group by rollup (a); + +*** ERROR[4384] GROUP BY ROLLUP clause not allowed for this statement. Reason: Distinct rollup aggregates not supported. + +*** ERROR[8822] The statement was not prepared. + +>> +>>select a,b,c,min(distinct b),sum(distinct d) from t033t1 group by rollup (a,b,c) order by 1,2,3; + +*** ERROR[4384] GROUP BY ROLLUP clause not allowed for this statement. Reason: Distinct rollup aggregates not supported. + +*** ERROR[8822] The statement was not prepared. + +>>select * from ( ++>select a,b,c,min(distinct b), sum(distinct d) from t033t1 group by (a,b,c) ++>union all ++>select a,b,cast(null as int),min(distinct b), sum(distinct d) from t033t1 group by (a,b) ++>union all ++>select a,cast(null as int),cast(null as int),min(distinct b), sum(distinct d) from t033t1 group by (a) ++>union all ++>select cast(null as int),cast(null as int),cast(null as int),min(distinct b), sum(distinct d) from t033t1 ++>) x(a,b,c,d,e) ++>order by a,b,c; + +A B C D E +----------- ----------- ----------- ----------- -------------------- + + 1 2 3 2 4 + 1 2 ? 2 4 + 1 3 3 3 3 + 1 3 ? 3 3 + 1 ? ? 2 7 + 2 3 4 3 5 + 2 3 ? 3 5 + 2 ? ? 3 5 + 3 3 3 3 3 + 3 3 4 3 5 + 3 3 ? 3 8 + 3 ? ? 3 8 + ? ? ? ? 6 + ? ? ? ? 6 + ? ? ? ? 6 + ? ? ? 2 18 + +--- 16 row(s) selected. +>> +>>select a+1, sum(b) from t033t1 group by rollup (1); + +(EXPR) (EXPR) +-------------------- -------------------- + + 2 5 + 3 3 + 4 6 + ? ? + ? 14 + +--- 5 row(s) selected. +>>select cast(d as int), sum(a) from t033t1 group by rollup (cast(d as int)); + +(EXPR) (EXPR) +----------- -------------------- + + 3 4 + 4 1 + 5 5 + 6 ? + ? 10 + +--- 5 row(s) selected. +>>select cast(d as nullable), sum(a) from t033t1 group by rollup (1); + +(EXPR) (EXPR) +----------- -------------------- + + 3 4 + 4 1 + 5 5 + 6 ? + ? 10 + +--- 5 row(s) selected. +>>select cast(d as nullable), sum(a) from t033t1 group by rollup (cast(d as nullable)); + +(EXPR) (EXPR) +----------- -------------------- + + 3 4 + 4 1 + 5 5 + 6 ? + ? 10 + +--- 5 row(s) selected. +>> +>>-- having clause +>>select a, sum(b) from t033t1 group by rollup (a) having a is null; + +A (EXPR) +----------- -------------------- + + ? ? + ? 14 + +--- 2 row(s) selected. +>>select a, sum(b) from t033t1 group by rollup (a) having a is not null; + +A (EXPR) +----------- -------------------- + + 1 5 + 2 3 + 3 6 + +--- 3 row(s) selected. +>> +>>-- partitioned table with esp execution +>>drop table if exists t033t2 cascade; + +--- SQL operation complete. +>>create table t033t2 (z int not null primary key, ++> a int, b int, c int, d int not null) ++> salt using 4 partitions; + +--- SQL operation complete. +>>insert into t033t2 values (10,1,2,3,4),(11,2,3,4,5),(12,3,3,3,3), ++> (13,3,3,4,5),(14,1,3,3,3),(15,null,null,null,6); + +--- 6 row(s) inserted. +>>control query shape esp_exchange(cut); + +--- SQL operation complete. +>>explain options 'f' select a,b,c,sum(d) from t033t2 ++> group by rollup(a,b,c) order by 1,2,3; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +6 . 7 root 8.00E+000 +5 . 6 esp_exchange 1:2(hash2) (m) 8.00E+000 +4 . 5 sort_partial_groupby 8.00E+000 +3 . 4 sort 8.00E+000 +2 . 3 esp_exchange 2(hash2):2(hash2) 8.00E+000 +1 . 2 hash_partial_groupby 8.00E+000 +. . 1 trafodion_scan T033T2 1.00E+002 + +--- SQL operation complete. +>>select a,b,c,sum(d) from t033t2 group by rollup(a,b,c) order by 1,2,3; + +A B C (EXPR) +----------- ----------- ----------- -------------------- + + 1 2 3 4 + 1 2 ? 4 + 1 3 3 3 + 1 3 ? 3 + 1 ? ? 3 + 1 ? ? 4 + 2 3 4 5 + 2 3 ? 5 + 2 ? ? 5 + 3 3 3 3 + 3 3 4 5 + 3 3 ? 5 + 3 3 ? 3 + 3 ? ? 3 + 3 ? ? 5 + ? ? ? 6 + ? ? ? 6 + ? ? ? 6 + ? ? ? 14 + ? ? ? 12 + +--- 20 row(s) selected. +>>control query shape cut; + +--- SQL operation complete. +>> +>>-- do not eliminate sort groupby rollup on primary keys +>>cqd allow_nullable_unique_key_constraint 'ON'; + +--- SQL operation complete. +>>create table t033t3 (a int primary key, b int); + +--- SQL operation complete. +>>insert into t033t3 values (1,1), (2,2), (null,null); + +--- 3 row(s) inserted. +>>prepare s from select a, sum(b) from t033t3 group by rollup (a) order by 1,2; + +--- SQL command prepared. +>>explain options 'f' s; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +3 . 4 root 1.00E+002 +2 . 3 sort 1.00E+002 +1 . 2 sort_groupby_rollup 1.00E+002 +. . 1 trafodion_scan T033T3 1.00E+002 + +--- SQL operation complete. +>>execute s; + +A (EXPR) +----------- -------------------- + + 1 1 + 2 2 + ? 3 + ? ? + +--- 4 row(s) selected. +>>-- regular groupby will eliminate sort groupby +>>prepare s from select a, sum(b) from t033t3 group by (a); + +--- SQL command prepared. +>>explain options 'f' s; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +1 . 2 root 1.00E+002 +. . 1 trafodion_scan T033T3 1.00E+002 + +--- SQL operation complete. +>> +>> +>>-- error cases. Not allowed or currently not supported +>>select a,a,sum(b) from t033t1 group by rollup (a,a); + +*** ERROR[4384] GROUP BY ROLLUP clause not allowed for this statement. Reason: Cannot have duplicate entries. + +*** ERROR[8822] The statement was not prepared. + +>>select d, sum(b) from t033t1 group by rollup (d); + +*** ERROR[4384] GROUP BY ROLLUP clause not allowed for this statement. Reason: Grouped columns must be nullable. + +*** ERROR[8822] The statement was not prepared. + +>> +>>log; diff --git a/core/sql/regress/seabase/TEST033 b/core/sql/regress/seabase/TEST033 new file mode 100644 index 0000000000..0d0bfdb0ed --- /dev/null +++ b/core/sql/regress/seabase/TEST033 @@ -0,0 +1,123 @@ +-- @@@ START COPYRIGHT @@@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- @@@ END COPYRIGHT @@@ +-- + +-- Tests for GROUP BY ROLLUP feature + +log LOG033 clear; + +drop table if exists t033t1 cascade; +drop table if exists t033t2 cascade; +drop table if exists t033t3 cascade; + +create table t033t1 (a int, b int, c int, d int not null); + +insert into t033t1 values (1,2,3,4),(2,3,4,5),(3,3,3,3),(3,3,4,5),(1,3,3,3), + (null,null,null,6); + +select * from t033t1 order by 1,2,3,4; + +explain options 'f' select a,b,c,sum(d) from t033t1 + group by rollup(a,b,c) order by 1,2,3; +select a,b,c,sum(d) from t033t1 group by rollup(a,b,c) order by 1,2,3; + +select * from ( +select a,b,c,sum(d) from t033t1 group by (a,b,c) +union all +select a,b,cast(null as int),sum(d) from t033t1 group by (a,b) +union all +select a,cast(null as int),cast(null as int),sum(d) from t033t1 group by (a) +union all +select cast(null as int),cast(null as int),cast(null as int),sum(d) from t033t1 +) x(a,b,c,d) +order by a,b,c; + +select b,c,a,sum(d) from t033t1 group by rollup(b,c,a) order by 1,2,3; + +select * from ( +select b,c,a,sum(d) from t033t1 group by (b,c,a) +union all +select b,c,cast(null as int),sum(d) from t033t1 group by (b,c) +union all +select b,cast(null as int),cast(null as int),sum(d) from t033t1 group by (b) +union all +select cast(null as int),cast(null as int),cast(null as int),sum(d) from t033t1 +) x(a,b,c,d) +order by a,b,c; + +select a,b,c from t033t1 group by rollup (a,b,c) order by 1,2,3; + +select a, count(distinct b) from t033t1 group by rollup (a); + +select a,b,c,min(distinct b),sum(distinct d) from t033t1 group by rollup (a,b,c) order by 1,2,3; +select * from ( +select a,b,c,min(distinct b), sum(distinct d) from t033t1 group by (a,b,c) +union all +select a,b,cast(null as int),min(distinct b), sum(distinct d) from t033t1 group by (a,b) +union all +select a,cast(null as int),cast(null as int),min(distinct b), sum(distinct d) from t033t1 group by (a) +union all +select cast(null as int),cast(null as int),cast(null as int),min(distinct b), sum(distinct d) from t033t1 +) x(a,b,c,d,e) +order by a,b,c; + +select a+1, sum(b) from t033t1 group by rollup (1); +select cast(d as int), sum(a) from t033t1 group by rollup (cast(d as int)); +select cast(d as nullable), sum(a) from t033t1 group by rollup (1); +select cast(d as nullable), sum(a) from t033t1 group by rollup (cast(d as nullable)); + +-- having clause +select a, sum(b) from t033t1 group by rollup (a) having a is null; +select a, sum(b) from t033t1 group by rollup (a) having a is not null; + +-- partitioned table with esp execution +drop table if exists t033t2 cascade; +create table t033t2 (z int not null primary key, + a int, b int, c int, d int not null) + salt using 4 partitions; +insert into t033t2 values (10,1,2,3,4),(11,2,3,4,5),(12,3,3,3,3), + (13,3,3,4,5),(14,1,3,3,3),(15,null,null,null,6); +control query shape esp_exchange(cut); +explain options 'f' select a,b,c,sum(d) from t033t2 + group by rollup(a,b,c) order by 1,2,3; +select a,b,c,sum(d) from t033t2 group by rollup(a,b,c) order by 1,2,3; +control query shape cut; + +-- do not eliminate sort groupby rollup on primary keys +cqd allow_nullable_unique_key_constraint 'ON'; +create table t033t3 (a int primary key, b int); +insert into t033t3 values (1,1), (2,2), (null,null); +prepare s from select a, sum(b) from t033t3 group by rollup (a) order by 1,2; +explain options 'f' s; +execute s; +-- regular groupby will eliminate sort groupby +prepare s from select a, sum(b) from t033t3 group by (a); +explain options 'f' s; + + +-- error cases. Not allowed or currently not supported +select a,a,sum(b) from t033t1 group by rollup (a,a); +select d, sum(b) from t033t1 group by rollup (d); + +log; + + + From 95b4a123c2d32bbbcff681ab34da450d5c067a3b Mon Sep 17 00:00:00 2001 From: Anoop Sharma Date: Thu, 29 Sep 2016 01:29:15 +0000 Subject: [PATCH 2/3] jira TRAFODION-2246 Add support for GROUP BY ROLLUP feature Syntax: select item1, item2... from ... group by rollup (v1,...,vn) Restrictions related to grouby rollup usage and query transformations listed below. Some may be lifted in later checkins: -- grouping column list must not have duplicate entries -- aggrs must be min/max/sum/avg/count/count(*) -- distinct aggregates are not supported -- grouping columns/values must be nullable. To group on a non-nullable item, use function CAST(v as nullable). This will convert v to nullable, if needed. -- HAVING predicate will not be pushed down to scan predicate. -- group by in a subquery will not be eliminated -- grouping comparison predicate does not use pcode -- group by rollup cannot be replaced by primary key -- groupby will be done using sortgroupby since ordering is needed to materialize rollup groups. -- rollup groupby will be done at root (master root or esp root). This is needed to detect rollup group changes across the whole result set. -- since groupby can materialize new null rows for rollup groups, a final sort operator is needed to be added after groupy even if sortgroupby has already guaranteed that it gets rows in order. Rollup computation logic is discussed in file executor/ex_sort_grby.cpp prior to ex_sort_grby_rollup_tcb::work method. See regress/seabase/TEST033 for tests of various 'group by rollup' functionality. --- core/sql/executor/ex_globals.h | 8 +- core/sql/executor/ex_sort_grby.cpp | 245 ++++++-------------- core/sql/executor/ex_sort_grby.h | 20 +- core/sql/exp/ExpPCodeClauseGen.cpp | 2 +- core/sql/generator/GenItemExpr.cpp | 4 +- core/sql/generator/GenPreCode.cpp | 67 +----- core/sql/generator/GenRelGrby.cpp | 12 +- core/sql/optimizer/BindRelExpr.cpp | 39 ++-- core/sql/optimizer/ItemExpr.h | 1 + core/sql/optimizer/ItemFunc.h | 1 + core/sql/optimizer/NormItemExpr.cpp | 8 +- core/sql/optimizer/NormRelExpr.cpp | 18 +- core/sql/optimizer/OptPhysRelExpr.cpp | 16 +- core/sql/optimizer/RelExpr.cpp | 6 +- core/sql/optimizer/RelGrby.h | 20 +- core/sql/regress/seabase/EXPECTED033 | 316 +++++++++++++++++++++----- core/sql/regress/seabase/TEST033 | 55 ++++- 17 files changed, 478 insertions(+), 360 deletions(-) diff --git a/core/sql/executor/ex_globals.h b/core/sql/executor/ex_globals.h index c9f20365e3..f835b1a870 100644 --- a/core/sql/executor/ex_globals.h +++ b/core/sql/executor/ex_globals.h @@ -283,10 +283,12 @@ NA_EIDPROC // This value is set when grouping expression to compute rollup is // evaluated. Caller (sort_grby_rollup_tcb) need to know the number // of the grouping column that caused a comparison to fail. - // This value is set during comp clause. Caller resets it before and + // This value is set during comp clause eval. Caller resets it before and // after call to expression evaluator. - // It is only a short duration setting. - // also see class ex_comp_clause in exp/exp_clause_derived.h on its usage. + // It is only a short duration global setting used to pass info from/to + // executor to expressions. + // also see class ex_comp_clause in exp/exp_clause_derived.h and ex_sort_grby_rollup_tcb + // on its usage. Int16 rollupColumnNum_; }; diff --git a/core/sql/executor/ex_sort_grby.cpp b/core/sql/executor/ex_sort_grby.cpp index 2c7692263f..1f3ffb455f 100644 --- a/core/sql/executor/ex_sort_grby.cpp +++ b/core/sql/executor/ex_sort_grby.cpp @@ -244,8 +244,7 @@ short ex_sort_grby_tcb::handleError(sort_grby_step &step, short &rc) } short ex_sort_grby_tcb::handleFinalize(sort_grby_step &step, - short &rc, - NABoolean noRelease) + short &rc) { ex_queue_entry * pentry_down = qparent_.down->getHeadEntry(); ex_sort_grby_private_state * pstate = @@ -314,8 +313,7 @@ short ex_sort_grby_tcb::handleFinalize(sort_grby_step &step, else step = SORT_GRBY_NEW_GROUP; - if (NOT noRelease) - workAtp_->getTupp(sort_grby_tdb().tuppIndex_).release(); + workAtp_->getTupp(sort_grby_tdb().tuppIndex_).release(); return 0; } @@ -389,6 +387,8 @@ short ex_sort_grby_tcb::handleDone(sort_grby_step &step, short &rc, //////////////////////////////////////////////////////////////////////////// short ex_sort_grby_tcb::work() { + short rc = 0; + // if no parent request, return if (qparent_.down->isEmpty()) return WORK_OK; @@ -468,43 +468,12 @@ short ex_sort_grby_tcb::work() { case ex_sort_grby_tcb::SORT_GRBY_CANCELLED: { - // request was cancelled. Child was sent a cancel - // request. Ignore all up rows from child. - // Wait for EOF. - if (qchild_.up->isEmpty()) - { - // nothing returned from child. Get outta here. - return WORK_OK; - } - - ex_queue_entry * centry = qchild_.up->getHeadEntry(); - - ex_assert(centry->upState.parentIndex == qparent_.down->getHeadIndex(), - "ex_sort_grby_tcb::work() child's reply out of sync"); - - ex_queue::up_status child_status = centry->upState.status; - switch(child_status) - { - case ex_queue::Q_OK_MMORE: - case ex_queue::Q_SQLERROR: - { - // just consume the child row - qchild_.up->removeHead(); - } - break; - - case ex_queue::Q_NO_DATA: - { - // return EOF to parent. - pstate->step_ = ex_sort_grby_tcb::SORT_GRBY_DONE; - } - break; + if (handleCancel(pstate->step_, rc)) + return rc; - case ex_queue::Q_INVALID: - ex_assert(0,"ex_sort_grby_tcb::work() Invalid state returned by child"); - break; + // next step set in handleCancel method + // step_ = SORT_GRBY_DONE - }; // end of switch on status of child queue } // request was cancelled break; @@ -603,7 +572,7 @@ short ex_sort_grby_tcb::work() } } // start of a new group - break; + break; case ex_sort_grby_tcb::SORT_GRBY_STARTED: { @@ -724,148 +693,37 @@ short ex_sort_grby_tcb::work() } break; - case SORT_GRBY_CHILD_ERROR: case SORT_GRBY_LOCAL_ERROR: { - qchild_.down->cancelRequestWithParentIndex( - qparent_.down->getHeadIndex()); - - // check if we've got room in the up queue - if (qparent_.up->isFull()) - return WORK_OK; // parent queue is full. Just return - - ex_queue_entry *pentry_up = qparent_.up->getTailEntry(); - ex_queue_entry * centry = qchild_.up->getHeadEntry(); - - pentry_up->copyAtp(centry); - pentry_up->upState.parentIndex = - pentry_down->downState.parentIndex; - pentry_up->upState.downIndex = qparent_.down->getHeadIndex(); - pentry_up->upState.setMatchNo(pstate->matchCount_); - - if ((sort_grby_tdb().isNonFatalErrorTolerated()) && - (SORT_GRBY_LOCAL_ERROR == pstate->step_)) - pentry_up->upState.status = ex_queue::Q_OK_MMORE; - else - pentry_up->upState.status = ex_queue::Q_SQLERROR; + if (handleError(pstate->step_, rc)) + return rc; - qparent_.up->insert(); - pstate->step_ = ex_sort_grby_tcb::SORT_GRBY_CANCELLED; - break; + // next step set in handleError method + // step_ = SORT_GRBY_CANCELLED } + break; case ex_sort_grby_tcb::SORT_GRBY_FINALIZE: case ex_sort_grby_tcb::SORT_GRBY_FINALIZE_CANCEL: { - // check if we've got room in the up queue - if (qparent_.up->isFull()) - return WORK_OK; // parent queue is full. Just return - - ex_expr::exp_return_type evalRetCode = ex_expr::EXPR_OK; - if (havingExpr()) - { - evalRetCode = havingExpr()->eval(workAtp_, 0); - } - - if ((!havingExpr()) || - ((havingExpr()) && (evalRetCode == ex_expr::EXPR_TRUE))) - { - // if stats are to be collected, collect them. - if (getStatsEntry()) - { - getStatsEntry()->incActualRowsReturned(); - } + if (handleFinalize(pstate->step_, rc)) + return rc; - // return to parent - ex_queue_entry * pentry_up = qparent_.up->getTailEntry(); - - pentry_up->copyAtp(workAtp_); - - pentry_up->upState.status = ex_queue::Q_OK_MMORE; - pentry_up->upState.parentIndex = pentry_down->downState.parentIndex; - pentry_up->upState.downIndex = qparent_.down->getHeadIndex(); - - pstate->matchCount_++; - pentry_up->upState.setMatchNo(pstate->matchCount_); - - // insert into parent up queue - qparent_.up->insert(); - } - else if (evalRetCode == ex_expr::EXPR_ERROR) - { - // The SORT_GRBY_LOCAL_ERROR state expects the - // diags area to be in the ATP of the head entry of - // the child's queue. It is currently in the workAtp_, - // so copy it to the childs head ATP. - // SORT_GRBY_LOCAL_ERROR will copy it from the - // child's queue entry to the parents up queue. - // - ex_queue_entry * centry = qchild_.up->getHeadEntry(); - centry->copyAtp(workAtp_); - pstate->step_ = SORT_GRBY_LOCAL_ERROR; - break; - } - - // start a new group, if more rows in child's up queue - if (pstate->step_ == SORT_GRBY_FINALIZE_CANCEL) - pstate->step_ = ex_sort_grby_tcb::SORT_GRBY_CANCELLED; - else - if (qchild_.up->getHeadEntry()->upState.status != - ex_queue::Q_OK_MMORE) - pstate->step_ = ex_sort_grby_tcb::SORT_GRBY_DONE; - else - pstate->step_ = ex_sort_grby_tcb::SORT_GRBY_NEW_GROUP; - - workAtp_->getTupp(sort_grby_tdb().tuppIndex_).release(); - - break; + // next step set in handleFinalize method. + // Could be one of the following: + // step_ = SORT_GRBY_LOCAL_ERROR + // SORT_GRBY_CANCELLED + // SORT_GRBY_NEW_GROUP + // SORT_GRBY_DONE } + break; case ex_sort_grby_tcb::SORT_GRBY_DONE: case ex_sort_grby_tcb::SORT_GRBY_NEVER_STARTED: { - // check if we've got room in the up queue - if (qparent_.up->isFull()) - return WORK_OK; // parent queue is full. Just return - - workAtp_->release(); - - ex_queue_entry * pentry_up = qparent_.up->getTailEntry(); - - pentry_up->upState.status = ex_queue::Q_NO_DATA; - pentry_up->upState.parentIndex = pentry_down->downState.parentIndex; - pentry_up->upState.downIndex = qparent_.down->getHeadIndex(); - pentry_up->upState.setMatchNo(pstate->matchCount_); - - if (pstate->step_ != SORT_GRBY_NEVER_STARTED) - { - ex_queue_entry * centry = qchild_.up->getHeadEntry(); - ex_assert(centry->upState.status == ex_queue::Q_NO_DATA, - "ex_sort_grby_tcb::work() expecting Q_NO_DATA"); - ex_assert(centry->upState.parentIndex == qparent_.down->getHeadIndex(), - "ex_sort_grby_tcb::work() child's reply out of sync"); - qchild_.up->removeHead(); - } - - // remove the down entry - qparent_.down->removeHead(); - - // insert into parent up queue - qparent_.up->insert(); - - // re-initialize pstate - pstate->step_ = ex_sort_grby_tcb::SORT_GRBY_EMPTY; - pstate->matchCount_ = 0; - workAtp_->release(); - - if (qparent_.down->isEmpty()) - return WORK_OK; - - // If we haven't given to our child the new head - // index return and ask to be called again. - if (qparent_.down->getHeadIndex() == processedInputs_) - return WORK_CALL_AGAIN; + if (handleDone(pstate->step_, rc)) + return rc; // postion at the new head pentry_down = qparent_.down->getHeadEntry(); @@ -906,7 +764,8 @@ ex_sort_grby_rollup_tcb::ex_sort_grby_rollup_tcb } } -short ex_sort_grby_rollup_tcb::rollupGrbyNull(Int16 groupNum) + +short ex_sort_grby_rollup_tcb::rollupGrbyMoveNull(Int16 groupNum) { ExpTupleDesc * returnRowTD = sort_grby_tdb().getCriDescUp()->getTupleDescriptor @@ -932,7 +791,7 @@ short ex_sort_grby_rollup_tcb::rollupGrbyNull(Int16 groupNum) return 0; } -short ex_sort_grby_rollup_tcb::rollupGrbyMove(ex_queue_entry * centry) +short ex_sort_grby_rollup_tcb::rollupGrbyMoveValue(ex_queue_entry * centry) { // move initial group by value for the regular aggr if (moveExpr()->eval(centry->getAtp(), workAtp_) == ex_expr::EXPR_ERROR) @@ -1182,7 +1041,7 @@ short ex_sort_grby_rollup_tcb::work() // move the group by value to this new row, if grouping // is being done. - if (rollupGrbyMove(centry)) + if (rollupGrbyMoveValue(centry)) { step_ = SORT_GRBY_LOCAL_ERROR; break; @@ -1200,8 +1059,41 @@ short ex_sort_grby_rollup_tcb::work() } else { - // no rows returned for a group by. - // Nothing to be returned(except, of course, Q_NO_DATA). + // no rows returned. + + // if aggrExpr, initialize aggragates for rollup group 0 and + // return it. + if (aggrExpr()) + { + // copy the down atp to the work atp for the current request + workAtp_->copyAtp(pentry_down->getAtp()); + + // get space to hold the grouped/aggregated row. + if (pool_->getFreeTuple(workAtp_->getTupp( + ((ex_sort_grby_tdb &)tdb).tuppIndex_))) + return WORK_POOL_BLOCKED; // couldn't allocate, try again. + + char * tempWorkDataPtr = + workAtp_->getTupp(sort_grby_tdb().tuppIndex_). + getDataPointer(); + workAtp_->getTupp(sort_grby_tdb().tuppIndex_) + .setDataPointer(rollupGroupAggrArr_[0]); + + if (((AggrExpr *)aggrExpr())->initializeAggr(workAtp_) == + ex_expr::EXPR_ERROR) + { + pstate->step_ = + SORT_GRBY_LOCAL_ERROR; + break; + } + + workAtp_->getTupp(sort_grby_tdb().tuppIndex_). + setDataPointer(tempWorkDataPtr); + + step_ = SORT_GRBY_ROLLUP_FINAL_GROUP; + break; + } + step_ = SORT_GRBY_DONE; } @@ -1329,7 +1221,6 @@ short ex_sort_grby_rollup_tcb::work() case SORT_GRBY_FINALIZE: case SORT_GRBY_FINALIZE_CANCEL: { - // if (handleFinalize(step_, rc, TRUE/*no workAtp release*/)) if (handleFinalize(step_, rc)) return rc; @@ -1400,7 +1291,7 @@ short ex_sort_grby_rollup_tcb::work() if (qparent_.up->isFull()) return WORK_OK; // parent queue is full. Just return - if (rollupGrbyNull(currGroupNum_)) + if (rollupGrbyMoveNull(currGroupNum_)) { step_ = SORT_GRBY_LOCAL_ERROR; break; @@ -1441,7 +1332,7 @@ short ex_sort_grby_rollup_tcb::work() return WORK_OK; // parent queue is full. Just return } - if (rollupGrbyNull(0)) + if (rollupGrbyMoveNull(0)) { step_ = SORT_GRBY_LOCAL_ERROR; break; @@ -1468,8 +1359,8 @@ short ex_sort_grby_rollup_tcb::work() if (handleDone(step_, rc, TRUE/*no child entry assertion check*/)) return rc; - // step_: set in handleDone method - // value: SORT_GRBY_EMPTY + // next step set in handleDone method + // step_ = SORT_GRBY_EMPTY return WORK_CALL_AGAIN; } // done diff --git a/core/sql/executor/ex_sort_grby.h b/core/sql/executor/ex_sort_grby.h index 65c0bfbfe4..bf1dc0744d 100644 --- a/core/sql/executor/ex_sort_grby.h +++ b/core/sql/executor/ex_sort_grby.h @@ -158,6 +158,7 @@ NA_EIDPROC SORT_GRBY_CHILD_ERROR, SORT_GRBY_NEVER_STARTED, + // next few states are for rollup sortgroupby tcb SORT_GRBY_ROLLUP_GROUPS_INIT, SORT_GRBY_ROLLUP_GROUP_START, SORT_GRBY_ROLLUP_GROUP, @@ -202,10 +203,13 @@ NA_EIDPROC } protected: + // code contained in the next 4 methods was previously part of mainline + // sort groupby code. + // That has now been extracted into these methods so it could be + // used by regular and rollup sortgroupby tcbs. short handleCancel(sort_grby_step &step, short &rc); short handleError(sort_grby_step &step, short &rc); - short handleFinalize(sort_grby_step &step, short &rc, - NABoolean noRelease = FALSE); + short handleFinalize(sort_grby_step &step, short &rc); short handleDone(sort_grby_step &step, short &rc, NABoolean noAssert = FALSE); @@ -231,10 +235,18 @@ class ex_sort_grby_rollup_tcb : public ex_sort_grby_tcb private: short rollupAggrInit(); - short rollupGrbyNull(Int16 groupNum); - short rollupGrbyMove(ex_queue_entry * centry); + + // move null to rollup array for all rollup groups impacted by change + // in group groupNum + short rollupGrbyMoveNull(Int16 groupNum); + + // move values from child row to rollup array + short rollupGrbyMoveValue(ex_queue_entry * centry); + + // evaluate aggregate for all regular and rollup entries short rollupAggrEval(ex_queue_entry * centry); + // array of entries where rollup aggrs and group values will be computed char ** rollupGroupAggrArr_; sort_grby_step step_; diff --git a/core/sql/exp/ExpPCodeClauseGen.cpp b/core/sql/exp/ExpPCodeClauseGen.cpp index 9bc6a182b6..d3a4e53605 100644 --- a/core/sql/exp/ExpPCodeClauseGen.cpp +++ b/core/sql/exp/ExpPCodeClauseGen.cpp @@ -808,7 +808,7 @@ ex_expr::exp_return_type ex_comp_clause::pCodeGenerate(Space *space, UInt32 f) { // comparison to fail, then do not generate pcode. That functionality is // not yet supported in pcode. // This is used for rollup group computation which need to know the particular - // group that caused comparison to fail. + // grouping column that caused the comparison to fail. if (getRollupColumnNum() >= 0) return ex_clause::pCodeGenerate(space, f); diff --git a/core/sql/generator/GenItemExpr.cpp b/core/sql/generator/GenItemExpr.cpp index 8c391ace51..ef904bd61e 100644 --- a/core/sql/generator/GenItemExpr.cpp +++ b/core/sql/generator/GenItemExpr.cpp @@ -633,11 +633,11 @@ short Convert::codeGen(Generator * generator) ex_conv_clause * conv_clause = new(generator->getSpace()) ex_conv_clause(getOperatorType(), attr, generator->getSpace()); - /* conv_clause->setLastVOAoffset(lastVOAOffset_); + conv_clause->setLastVOAoffset(lastVOAOffset_); conv_clause->setLastNullIndicatorLength(lastNullIndicatorLength_); conv_clause->setLastVcIndicatorLength(lastVcIndicatorLength_); conv_clause->setAlignment(alignment_); - */ + generator->getExpGenerator()->linkClause(this, conv_clause); diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp index 3b417d2f3c..8d8ab9c5ca 100644 --- a/core/sql/generator/GenPreCode.cpp +++ b/core/sql/generator/GenPreCode.cpp @@ -5744,56 +5744,6 @@ RelExpr * HashGroupBy::preCodeGen(Generator * generator, return GroupByAgg::preCodeGen(generator, externalInputs, pulledNewInputs); } - -RelExpr *GroupByAgg::processGroupbyRollup(Generator * generator, - const ValueIdSet & externalInputs, - ValueIdSet &pulledNewInputs) -{ - // if not rollup or no groupby or num of grouping columns is 0, return. - if ((! isRollup()) || - (groupExprList().entries() == 0)) - return this; - - ValueId valId; - - // rollup groups need to be evaluated in the order that they were specified - // in the query. - // groupExprList() contains grouping items in user specified order. - // For each item in groupExpr() set, find its position in groupExprList(). - // Create a new list corresponding to the rearraned groupExpr() set. This list - // will be used during codeGen to generate rollup group expression. - for (valId = groupExpr().init(); - groupExpr().next(valId); - groupExpr().advance(valId)) - { - ItemExpr * itemExpr = valId.getItemExpr(); - - if (itemExpr->getOperatorType() != ITM_INDEXCOLUMN) - continue; - - NAColumn * nac = ((IndexColumn*)itemExpr)->getNAColumn(); - - NABoolean found = FALSE; - for (CollIndex ii = 0; - (NOT found && ii < groupExprList().entries()); ii++) - { - ValueId valId2 = groupExprList()[ii]; - ItemExpr * itemExpr2 = valId2.getItemExpr(); - if (itemExpr2->getOperatorType() != ITM_BASECOLUMN) - continue; - - NAColumn * nac2 = ((BaseColumn*)itemExpr2)->getNAColumn(); - found = (nac->getColName() == nac2->getColName()); - if (found) - { - groupExprList()[ii] = valId; - } - } // inner for - } // outer for - - return this; -} - RelExpr * GroupByAgg::preCodeGen(Generator * generator, const ValueIdSet & externalInputs, ValueIdSet &pulledNewInputs) @@ -5868,6 +5818,15 @@ RelExpr * GroupByAgg::preCodeGen(Generator * generator, replicatePredicates, &getGroupAttr()->getCharacteristicOutputs()); + // Rebuild the rollup grouping expressions tree. Use bridge values, if possible + rollupGroupExprList().replaceVEGExpressions + (availableValues, + getGroupAttr()->getCharacteristicInputs(), + FALSE, // No key predicates need to be generated here + NULL, + replicatePredicates, + &getGroupAttr()->getCharacteristicOutputs()); + // Rebuild the aggregate expressions tree aggregateExpr().replaceVEGExpressions (availableValues, @@ -5928,14 +5887,6 @@ RelExpr * GroupByAgg::preCodeGen(Generator * generator, } - if (isRollup()) - { - RelExpr * gbrExpr = processGroupbyRollup - (generator, externalInputs, pulledNewInputs); - if (! gbrExpr) - return NULL; - } - markAsPreCodeGenned(); // Done. diff --git a/core/sql/generator/GenRelGrby.cpp b/core/sql/generator/GenRelGrby.cpp index 9b549a03e6..bed0c6f5a1 100644 --- a/core/sql/generator/GenRelGrby.cpp +++ b/core/sql/generator/GenRelGrby.cpp @@ -67,7 +67,7 @@ short GroupByAgg::genAggrGrbyExpr(Generator * generator, ValueIdSet &aggregateExpr, ValueIdSet &groupExpr, - ValueIdList &groupExprList, + ValueIdList &rollupGroupExprList, ValueIdSet &selectionPred, Int32 workAtp, Int32 workAtpIndex, @@ -284,9 +284,9 @@ short GroupByAgg::genAggrGrbyExpr(Generator * generator, ValueIdList gbyValIdList; ValueIdSet searchValIdSet; - if (isRollup() && (NOT groupExprList.isEmpty())) { - for (CollIndex j = 0; j < groupExprList.entries(); j++) { - valId = groupExprList[j]; + if (isRollup() && (NOT rollupGroupExprList.isEmpty())) { + for (CollIndex j = 0; j < rollupGroupExprList.entries(); j++) { + valId = rollupGroupExprList[j]; ItemExpr * itemExpr = valId.getItemExpr(); @@ -1777,7 +1777,7 @@ short GroupByAgg::codeGen(Generator * generator) { genAggrGrbyExpr(generator, aggregateExpr(), groupExpr(), - groupExprList(), + rollupGroupExprList(), selectionPred(), 1, returnedDesc->noTuples() - 1, @@ -1821,7 +1821,7 @@ short GroupByAgg::codeGen(Generator * generator) { { sortGrbyTdb->setIsRollup(TRUE); - sortGrbyTdb->setNumRollupGroups(groupExprList().entries()); + sortGrbyTdb->setNumRollupGroups(rollupGroupExprList().entries()); } if(!generator->explainDisabled()) { diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index 7f479a22c5..71f3b0b629 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -4328,7 +4328,7 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase2(BindWA *bindWA) if (grby->isRollup()) { - if (grby->groupExpr().entries() != grby->groupExprList().entries()) + if (grby->groupExpr().entries() != grby->rollupGroupExprList().entries()) { *CmpCommon::diags() << DgSqlCode(-4384) << DgString0("Cannot have duplicate entries."); @@ -4396,8 +4396,11 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase2(BindWA *bindWA) groupExprCpy.remove(vid); groupExprCpy.insert(grpById); - CollIndex idx = grby->groupExprList().index(vid); - grby->groupExprList()[idx] = grpById; + if (grby->isRollup()) + { + CollIndex idx = grby->rollupGroupExprList().index(vid); + grby->rollupGroupExprList()[idx] = grpById; + } } else { //sequence @@ -4521,9 +4524,9 @@ RelRoot * RelRoot::transformGroupByWithOrdinalPhase2(BindWA *bindWA) ValueId valId; if (grby->isRollup()) { - for (CollIndex i = 0; i < grby->groupExprList().entries(); i++) + for (CollIndex i = 0; i < grby->rollupGroupExprList().entries(); i++) { - valId = grby->groupExprList()[i]; + valId = grby->rollupGroupExprList()[i]; if (NOT valId.getType().supportsSQLnull()) { @@ -6950,17 +6953,20 @@ RelExpr *GroupByAgg::bindNode(BindWA *bindWA) ItemExpr *groupExprTree = removeGroupExprTree(); if (groupExprTree) { currScope->context()->inGroupByClause() = TRUE; - groupExprTree->convertToValueIdList(groupExprList(), bindWA, ITM_ITEM_LIST); + groupExprTree->convertToValueIdSet(groupExpr(), bindWA, ITM_ITEM_LIST); + + if (isRollup()) + groupExprTree->convertToValueIdList( + rollupGroupExprList(), bindWA, ITM_ITEM_LIST); currScope->context()->inGroupByClause() = FALSE; if (bindWA->errStatus()) return this; - ValueIdSet vidSet(groupExprList()); - setGroupExpr(vidSet); - - for (CollIndex i = 0; i < groupExprList().entries(); i++) + ValueIdList groupByList(groupExpr()); + + for (CollIndex i = 0; i < groupByList.entries(); i++) { - ValueId vid = groupExprList()[i]; + ValueId vid = groupByList[i]; vid.getItemExpr()->setIsGroupByExpr(TRUE); } @@ -6970,9 +6976,9 @@ RelExpr *GroupByAgg::bindNode(BindWA *bindWA) RETDesc * childRETDesc = child(0)->getRETDesc(); ItemExprList origSelectList(getParentRootSelectList(), bindWA->wHeap()); - for (CollIndex i = 0; i < groupExprList().entries(); i++) + for (CollIndex i = 0; i < groupByList.entries(); i++) { - ValueId vid = groupExprList()[i]; + ValueId vid = groupByList[i]; if((vid.getItemExpr()->getOperatorType() == ITM_SEL_INDEX)&& (((SelIndex*)(vid.getItemExpr()))->renamedColNameInGrbyClause())) { @@ -6988,6 +6994,13 @@ RelExpr *GroupByAgg::bindNode(BindWA *bindWA) { groupExpr().remove(vid); groupExpr().insert(baseColExpr->getValueId()); + + if (isRollup()) + { + CollIndex idx = rollupGroupExprList().index(vid); + rollupGroupExprList()[idx] = baseColExpr->getValueId(); + } + baseColExpr->getColumnDesc()->setGroupedFlag(); origSelectList[indx]->setInGroupByOrdinal(FALSE); } diff --git a/core/sql/optimizer/ItemExpr.h b/core/sql/optimizer/ItemExpr.h index 7add002c90..9ad70352d1 100644 --- a/core/sql/optimizer/ItemExpr.h +++ b/core/sql/optimizer/ItemExpr.h @@ -1282,6 +1282,7 @@ class ItemExpr : public ExprNode WAS_DEFAULT_CLAUSE = 0x0040, // if set, the subtree rooted below was part of "groupby rollup" clause. + // Currently used during parsing phase. See parser/sqlparser.y. IS_GROUPBY_ROLLUP = 0x0080 }; diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h index 10086d354d..a922fcd6fd 100644 --- a/core/sql/optimizer/ItemFunc.h +++ b/core/sql/optimizer/ItemFunc.h @@ -2565,6 +2565,7 @@ class CastType : public Cast virtual short codeGen(Generator*); private: + // if set, then cast child to nullable type NABoolean makeNullable_; }; // CastType diff --git a/core/sql/optimizer/NormItemExpr.cpp b/core/sql/optimizer/NormItemExpr.cpp index 6fcb8680f6..a911744cd1 100644 --- a/core/sql/optimizer/NormItemExpr.cpp +++ b/core/sql/optimizer/NormItemExpr.cpp @@ -3135,9 +3135,13 @@ void Subquery::transformToRelExpr(NormWA & normWARef, GroupByAgg *aggNode = (GroupByAgg *)childOfRoot; // If the group by has a group by list and no aggregate - // functions we can eliminate it + // functions we can eliminate it. Keep ROLLUP groupbys + // for now. This could later be improved, if we have + // "null-rejecting" predicates that exclude the extra + // rows included by the rollup. if (!aggNode->groupExpr().isEmpty() && - aggNode->aggregateExpr().isEmpty()) + aggNode->aggregateExpr().isEmpty() && + !aggNode->isRollup()) { // Remove the aggNode and pass the selection predicates // and inputs to the new child of the Root diff --git a/core/sql/optimizer/NormRelExpr.cpp b/core/sql/optimizer/NormRelExpr.cpp index 2c623503fc..539179b6f2 100644 --- a/core/sql/optimizer/NormRelExpr.cpp +++ b/core/sql/optimizer/NormRelExpr.cpp @@ -5216,6 +5216,12 @@ void GroupByAgg::rewriteNode(NormWA & normWARef) { } // --------------------------------------------------------------------- + // Rewrite the expressions that are rollup grouping expressions + // --------------------------------------------------------------------- + if (rollupGroupExprList().normalizeNode(normWARef)) + { + } + // --------------------------------------------------------------------- // Rewrite the expressions that are aggregate expressions // --------------------------------------------------------------------- if (aggregateExpr().normalizeNode(normWARef)) @@ -5346,15 +5352,17 @@ RelExpr * GroupByAgg::normalizeNode(NormWA & normWARef) // --------------------------------------------------------------------- // if this is a rollup groupby, then do not pushdown having pred to - // child node. If pushdown is done, then it might eliminate rows that + // child node. If pushdown is done, then it might incorrectly process rows that // are generated during rollup groupby processing. // For ex: + // insert into t values (1); // select a from t group by rollup(a) having a is not null; - // If 'having' pred is pushdown to scan node, then it will not return - // any rows from scan and will elimite processing of rollup groups - // that are returned by groupby. + // If 'having' pred is pushdown to scan node as a where pred, + // then SortGroupBy will return all rollup groups generated + // and represented as null. They will not be filtered out which + // they would if having pred is applied after rollup group materialization. // Maybe later we can optimize so this pushdown is done if possible, - // for ex, if there are no 'is null/not null' preds on grouping cols. + // for ex, if there are no 'is null/not null' having preds on grouping cols. if (NOT isRollup()) { pushdownCoveredExpr(getGroupAttr()->getCharacteristicOutputs(), diff --git a/core/sql/optimizer/OptPhysRelExpr.cpp b/core/sql/optimizer/OptPhysRelExpr.cpp index 1acfd76b1b..61969e94d0 100644 --- a/core/sql/optimizer/OptPhysRelExpr.cpp +++ b/core/sql/optimizer/OptPhysRelExpr.cpp @@ -11680,7 +11680,12 @@ Context* GroupByAgg::createContextForAChild(Context* myContext, // Also don't specify a required number of partitions for a scalar // aggregate, because a scalar aggregate cannot execute in parallel. // --------------------------------------------------------------------- - if ( !isAPartialGroupByLeaf1() && !isAPartialGroupByLeaf2() && + if (isRollup()) + { + // GROUP BY ROLLUP needs to be evaluated in a single process + rg.addNumOfPartitions(1); + } + else if ( !isAPartialGroupByLeaf1() && !isAPartialGroupByLeaf2() && okToAttemptESPParallelism(myContext, pws, childNumPartsRequirement, @@ -11800,7 +11805,12 @@ void SortGroupBy::addArrangementAndOrderRequirements( // columns. // Once we have "partial" arrangement requirements we // could indicate those for partial groupbys. - if (NOT groupExpr().isEmpty()) + // A GROUP BY ROLLUP needs the exact order as specified. + if (isRollup() && (NOT rollupGroupExprList().isEmpty())) + { + rg.addSortKey(rollupGroupExprList()); + } + else if (NOT groupExpr().isEmpty()) { // Shouldn't/Can't add a sort order type requirement // if we are in DP2 @@ -11848,7 +11858,7 @@ SortGroupBy::synthPhysicalProperty(const Context* myContext, PhysicalProperty* sppForMe = new (CmpCommon::statementHeap()) - PhysicalProperty(sppOfChild->getSortKey(), + PhysicalProperty((isRollup() ? ValueIdList() : sppOfChild->getSortKey()), sppOfChild->getSortOrderType(), sppOfChild->getDp2SortOrderPartFunc(), sppOfChild->getPartitioningFunction(), diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp index 3c2363454b..9aae591f3d 100644 --- a/core/sql/optimizer/RelExpr.cpp +++ b/core/sql/optimizer/RelExpr.cpp @@ -7560,7 +7560,7 @@ RelExpr * GroupByAgg::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) result->aggregateExprTree_ = aggregateExprTree_->copyTree(outHeap); result->groupExpr_ = groupExpr_; - result->groupExprList_ = groupExprList_; + result->rollupGroupExprList_ = rollupGroupExprList_; result->aggregateExpr_ = aggregateExpr_; result->formEnum_ = formEnum_; result->gbAggPushedBelowTSJ_ = gbAggPushedBelowTSJ_; @@ -7712,8 +7712,8 @@ void GroupByAgg::addLocalExpr(LIST(ExprNode *) &xlist, { if (groupExpr_.isEmpty()) xlist.insert(groupExprTree_); - else if (isRollup() && (NOT groupExprList_.isEmpty())) - xlist.insert(groupExprList_.rebuildExprTree(ITM_ITEM_LIST)); + else if (isRollup() && (NOT rollupGroupExprList_.isEmpty())) + xlist.insert(rollupGroupExprList_.rebuildExprTree(ITM_ITEM_LIST)); else xlist.insert(groupExpr_.rebuildExprTree(ITM_ITEM_LIST)); llist.insert("grouping_columns"); diff --git a/core/sql/optimizer/RelGrby.h b/core/sql/optimizer/RelGrby.h index 9a644ae415..72717fb03f 100644 --- a/core/sql/optimizer/RelGrby.h +++ b/core/sql/optimizer/RelGrby.h @@ -152,14 +152,11 @@ class GroupByAgg : public RelExpr inline void setGroupExpr(ValueIdSet &expr) { groupExpr_ = expr;} inline void addGroupExpr(ValueIdSet &expr) { groupExpr_ += expr;} - ValueIdList & groupExprList() { return groupExprList_; } - const ValueIdList & groupExprList() const { return groupExprList_; } - void setGroupExprList(ValueIdList &expr) { groupExprList_ = expr;} + ValueIdList & rollupGroupExprList() { return rollupGroupExprList_; } + const ValueIdList & rollupGroupExprList() const { return rollupGroupExprList_; } + void setRollupGroupExprList(ValueIdList &expr) { rollupGroupExprList_ = expr;} - inline ValueIdList & rollupAggrExpr() { return rollupAggrExpr_; } - inline const ValueIdList & rollupAggrExpr() const { return rollupAggrExpr_; } - - // return a (short-lived) read/write reference to the item expressions + // return a (short-lived) read/write reference to the item expressions inline ValueIdSet & leftUniqueExpr() { return leftUniqueExpr_; } inline const ValueIdSet & leftUniqueExpr() const { return leftUniqueExpr_; } inline void setLeftUniqueExpr(ValueIdSet &expr) { leftUniqueExpr_ = expr;} @@ -410,7 +407,7 @@ class GroupByAgg : public RelExpr short genAggrGrbyExpr(Generator * generator, ValueIdSet &aggregateExpr, ValueIdSet &groupExpr, - ValueIdList &groupExprList, + ValueIdList &rollupGroupExprList, ValueIdSet &selectionPred, Int32 workAtp, Int32 workAtpIndex, short returnedAtpIndex, @@ -527,10 +524,6 @@ class GroupByAgg : public RelExpr ValueIdSet &pulledPredicates, // return the pulled-up preds ValueIdMap *optionalMap); // optional map to rewrite preds - RelExpr *processGroupbyRollup(Generator * generator, - const ValueIdSet & externalInputs, - ValueIdSet &pulledNewInputs); - void setIsRollup(NABoolean v) { isRollup_ = v; } NABoolean isRollup() { return isRollup_; } const NABoolean isRollup() const { return isRollup_; } @@ -563,8 +556,7 @@ class GroupByAgg : public RelExpr // -------------------------------------- // used for processing groupby rollup // -------------------------------------- - ValueIdList groupExprList_; - ValueIdList rollupAggrExpr_; + ValueIdList rollupGroupExprList_; // --------------------------------------------------------------------- // The expression specifying the aggregates to be generated diff --git a/core/sql/regress/seabase/EXPECTED033 b/core/sql/regress/seabase/EXPECTED033 index f1ec8e07cc..754855c2ac 100644 --- a/core/sql/regress/seabase/EXPECTED033 +++ b/core/sql/regress/seabase/EXPECTED033 @@ -13,6 +13,24 @@ --- SQL operation complete. >> +>>-- empty table +>>select a, sum(b) from t033t1 group by rollup (a); + +A (EXPR) +----------- -------------------- + + ? ? + +--- 1 row(s) selected. +>>select b+1, sum(b) from t033t1 group by rollup (b); + +(EXPR) (EXPR) +-------------------- -------------------- + + ? ? + +--- 1 row(s) selected. +>> >>insert into t033t1 values (1,2,3,4),(2,3,4,5),(3,3,3,3),(3,3,4,5),(1,3,3,3), +> (null,null,null,6); @@ -38,66 +56,67 @@ A B C D LC RC OP OPERATOR OPT DESCRIPTION CARD ---- ---- ---- -------------------- -------- -------------------- --------- -3 . 4 root 8.00E+000 +4 . 5 root 8.00E+000 +3 . 4 sort 8.00E+000 2 . 3 sort_groupby_rollup 8.00E+000 1 . 2 sort 1.00E+002 . . 1 trafodion_scan T033T1 1.00E+002 --- SQL operation complete. ->>select a,b,c,sum(d) from t033t1 group by rollup(a,b,c) order by 1,2,3; - -A B C (EXPR) ------------ ----------- ----------- -------------------- - - 1 2 3 4 - 1 2 ? 4 - 1 3 3 3 - 1 3 ? 3 - 1 ? ? 7 - 2 3 4 5 - 2 3 ? 5 - 2 ? ? 5 - 3 3 3 3 - 3 3 4 5 - 3 3 ? 8 - 3 ? ? 8 - ? ? ? 6 - ? ? ? 6 - ? ? ? 6 - ? ? ? 26 +>>select a,b,c,min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 group by rollup(a,b,c) order by 1,2,3,4,5,6,7,8,9,10; + +A B C (EXPR) (EXPR) (EXPR) (EXPR) (EXPR) (EXPR) (EXPR) +----------- ----------- ----------- ----------- ----------- -------------------- -------------------- -------------------- -------------------- -------------------- + + 1 2 3 4 4 4 4 1 1 1 + 1 2 ? 4 4 4 4 1 1 1 + 1 3 3 3 3 3 3 1 1 1 + 1 3 ? 3 3 3 3 1 1 1 + 1 ? ? 3 4 7 3 2 2 2 + 2 3 4 5 5 5 5 1 1 1 + 2 3 ? 5 5 5 5 1 1 1 + 2 ? ? 5 5 5 5 1 1 1 + 3 3 3 3 3 3 3 1 1 1 + 3 3 4 5 5 5 5 1 1 1 + 3 3 ? 3 5 8 4 2 2 2 + 3 ? ? 3 5 8 4 2 2 2 + ? ? ? 3 6 26 4 6 6 5 + ? ? ? 6 6 6 6 1 1 0 + ? ? ? 6 6 6 6 1 1 0 + ? ? ? 6 6 6 6 1 1 0 --- 16 row(s) selected. >> >>select * from ( -+>select a,b,c,sum(d) from t033t1 group by (a,b,c) ++>select a,b,c,min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 group by (a,b,c) +>union all -+>select a,b,cast(null as int),sum(d) from t033t1 group by (a,b) ++>select a,b,cast(null as int),min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 group by (a,b) +>union all -+>select a,cast(null as int),cast(null as int),sum(d) from t033t1 group by (a) ++>select a,cast(null as int),cast(null as int),min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 group by (a) +>union all -+>select cast(null as int),cast(null as int),cast(null as int),sum(d) from t033t1 -+>) x(a,b,c,d) -+>order by a,b,c; - -A B C D ------------ ----------- ----------- -------------------- - - 1 2 3 4 - 1 2 ? 4 - 1 3 3 3 - 1 3 ? 3 - 1 ? ? 7 - 2 3 4 5 - 2 3 ? 5 - 2 ? ? 5 - 3 3 3 3 - 3 3 4 5 - 3 3 ? 8 - 3 ? ? 8 - ? ? ? 6 - ? ? ? 6 - ? ? ? 6 - ? ? ? 26 ++>select cast(null as int),cast(null as int),cast(null as int),min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 ++>) x(a,b,c,d,e,f,g,h,i,j) ++>order by a,b,c,d,e,f,g,h,i,j; + +A B C D E F G H I J +----------- ----------- ----------- ----------- ----------- -------------------- -------------------- -------------------- -------------------- -------------------- + + 1 2 3 4 4 4 4 1 1 1 + 1 2 ? 4 4 4 4 1 1 1 + 1 3 3 3 3 3 3 1 1 1 + 1 3 ? 3 3 3 3 1 1 1 + 1 ? ? 3 4 7 3 2 2 2 + 2 3 4 5 5 5 5 1 1 1 + 2 3 ? 5 5 5 5 1 1 1 + 2 ? ? 5 5 5 5 1 1 1 + 3 3 3 3 3 3 3 1 1 1 + 3 3 4 5 5 5 5 1 1 1 + 3 3 ? 3 5 8 4 2 2 2 + 3 ? ? 3 5 8 4 2 2 2 + ? ? ? 3 6 26 4 6 6 5 + ? ? ? 6 6 6 6 1 1 0 + ? ? ? 6 6 6 6 1 1 0 + ? ? ? 6 6 6 6 1 1 0 --- 16 row(s) selected. >> @@ -132,7 +151,7 @@ B C A (EXPR) +>union all +>select cast(null as int),cast(null as int),cast(null as int),sum(d) from t033t1 +>) x(a,b,c,d) -+>order by a,b,c; ++>order by 1,2,3; A B C D ----------- ----------- ----------- -------------------- @@ -271,6 +290,46 @@ A B C D E 6 ? ? 10 +--- 5 row(s) selected. +>> +>>-- firstN and cancel processing +>>select [first 1] a,b,sum(c) from t033t1 group by rollup(1,2); + +A B (EXPR) +----------- ----------- -------------------- + + 1 2 3 + +--- 1 row(s) selected. +>>select [first 9] a,b,sum(c) from t033t1 group by rollup(1,2); + +A B (EXPR) +----------- ----------- -------------------- + + 1 2 3 + 1 3 3 + 1 ? 6 + 2 3 4 + 2 ? 4 + 3 3 7 + 3 ? 7 + ? ? ? + ? ? ? + +--- 9 row(s) selected. +>> +>>-- order by descending +>>select a from t033t1 group by rollup(a) order by a desc; + +A +----------- + + ? + ? + 3 + 2 + 1 + --- 5 row(s) selected. >> >>-- having clause @@ -294,6 +353,143 @@ A (EXPR) --- 3 row(s) selected. >> +>>-- grby rollup and joins and subqueries +>>prepare s from ++>select * from (select a,sum(b) from t033t1 group by rollup(a)) x(a1,b1), ++> (select a,sum(b) from t033t1 group by rollup(a)) y(a2,b2) ++> where x.a1 = y.a2; + +--- SQL command prepared. +>>explain options 'f' s; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +7 . 8 root 2.00E+000 +6 3 7 hybrid_hash_join u 2.00E+000 +5 . 6 sort_groupby_rollup 2.00E+000 +4 . 5 sort 1.00E+002 +. . 4 trafodion_scan T033T1 1.00E+002 +2 . 3 sort_groupby_rollup 2.00E+000 +1 . 2 sort 1.00E+002 +. . 1 trafodion_scan T033T1 1.00E+002 + +--- SQL operation complete. +>>execute s; + +A1 B1 A2 B2 +----------- -------------------- ----------- -------------------- + + 1 5 1 5 + 2 3 2 3 + 3 6 3 6 + +--- 3 row(s) selected. +>> +>>prepare s from ++>select * from (select a,sum(b) from t033t1 group by rollup(a)) x(a1,b1), ++> (select a,sum(b) from t033t1 group by rollup(a)) y(a2,b2) ++> where x.a1 = y.a2 or (x.a1 is null and y.a2 is null); + +--- SQL command prepared. +>>explain options 'f' s; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +7 . 8 root 3.00E+000 +6 3 7 hybrid_hash_join 3.00E+000 +5 . 6 sort_groupby_rollup 2.00E+000 +4 . 5 sort 1.00E+002 +. . 4 trafodion_scan T033T1 1.00E+002 +2 . 3 sort_groupby_rollup 2.00E+000 +1 . 2 sort 1.00E+002 +. . 1 trafodion_scan T033T1 1.00E+002 + +--- SQL operation complete. +>>execute s; + +A1 B1 A2 B2 +----------- -------------------- ----------- -------------------- + + 1 5 1 5 + 2 3 2 3 + 3 6 3 6 + ? ? ? ? + ? ? ? 14 + ? 14 ? ? + ? 14 ? 14 + +--- 7 row(s) selected. +>> +>>-- should not convert subq to join and eliminate groupby rollup +>>prepare s from ++>select a from t033t1 where a in (select a from t033t1 group by rollup(a)) ++> or a is null order by a; + +--- SQL command prepared. +>>explain options 'f' s; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +8 . 9 root 5.09E+001 +7 . 8 sort 5.09E+001 +1 6 7 nested_join 5.09E+001 +5 . 6 probe_cache 5.09E-001 +4 . 5 sort_scalar_aggr 5.09E-001 +3 . 4 sort_groupby_rollup 2.00E+000 +2 . 3 sort 1.00E+002 +. . 2 trafodion_scan T033T1 1.00E+002 +. . 1 trafodion_scan T033T1 1.00E+002 + +--- SQL operation complete. +>>execute s; + +A +----------- + + 1 + 1 + 2 + 3 + 3 + ? + +--- 6 row(s) selected. +>> +>>prepare s from ++>select a from t033t1 where a in (select NVL(a, 2) from t033t1 group by rollup(a)) ++> order by 1; + +--- SQL command prepared. +>>explain options 'f' s; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +6 . 7 root 3.29E+001 +5 . 6 sort 3.29E+001 +4 3 5 hybrid_hash_semi_joi u 3.29E+001 +. . 4 trafodion_scan T033T1 1.00E+002 +2 . 3 sort_groupby_rollup 2.00E+000 +1 . 2 sort 1.00E+002 +. . 1 trafodion_scan T033T1 1.00E+002 + +--- SQL operation complete. +>>execute s; + +A +----------- + + 1 + 1 + 2 + 3 + 3 + +--- 5 row(s) selected. +>> >>-- partitioned table with esp execution >>drop table if exists t033t2 cascade; @@ -316,11 +512,13 @@ A (EXPR) LC RC OP OPERATOR OPT DESCRIPTION CARD ---- ---- ---- -------------------- -------- -------------------- --------- -6 . 7 root 8.00E+000 -5 . 6 esp_exchange 1:2(hash2) (m) 8.00E+000 +8 . 9 root 8.00E+000 +7 . 8 esp_exchange 1:2(hash2) (m) 8.00E+000 +6 . 7 sort 8.00E+000 +5 . 6 esp_exchange 2(hash2):1 8.00E+000 4 . 5 sort_partial_groupby 8.00E+000 3 . 4 sort 8.00E+000 -2 . 3 esp_exchange 2(hash2):2(hash2) 8.00E+000 +2 . 3 esp_exchange 1:2(hash2) 8.00E+000 1 . 2 hash_partial_groupby 8.00E+000 . . 1 trafodion_scan T033T2 1.00E+002 @@ -334,24 +532,20 @@ A B C (EXPR) 1 2 ? 4 1 3 3 3 1 3 ? 3 - 1 ? ? 3 - 1 ? ? 4 + 1 ? ? 7 2 3 4 5 2 3 ? 5 2 ? ? 5 3 3 3 3 3 3 4 5 - 3 3 ? 5 - 3 3 ? 3 - 3 ? ? 3 - 3 ? ? 5 + 3 3 ? 8 + 3 ? ? 8 ? ? ? 6 ? ? ? 6 ? ? ? 6 - ? ? ? 14 - ? ? ? 12 + ? ? ? 26 ---- 20 row(s) selected. +--- 16 row(s) selected. >>control query shape cut; --- SQL operation complete. diff --git a/core/sql/regress/seabase/TEST033 b/core/sql/regress/seabase/TEST033 index 0d0bfdb0ed..4cd7e48984 100644 --- a/core/sql/regress/seabase/TEST033 +++ b/core/sql/regress/seabase/TEST033 @@ -30,6 +30,10 @@ drop table if exists t033t3 cascade; create table t033t1 (a int, b int, c int, d int not null); +-- empty table +select a, sum(b) from t033t1 group by rollup (a); +select b+1, sum(b) from t033t1 group by rollup (b); + insert into t033t1 values (1,2,3,4),(2,3,4,5),(3,3,3,3),(3,3,4,5),(1,3,3,3), (null,null,null,6); @@ -37,18 +41,18 @@ select * from t033t1 order by 1,2,3,4; explain options 'f' select a,b,c,sum(d) from t033t1 group by rollup(a,b,c) order by 1,2,3; -select a,b,c,sum(d) from t033t1 group by rollup(a,b,c) order by 1,2,3; +select a,b,c,min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 group by rollup(a,b,c) order by 1,2,3,4,5,6,7,8,9,10; select * from ( -select a,b,c,sum(d) from t033t1 group by (a,b,c) +select a,b,c,min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 group by (a,b,c) union all -select a,b,cast(null as int),sum(d) from t033t1 group by (a,b) +select a,b,cast(null as int),min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 group by (a,b) union all -select a,cast(null as int),cast(null as int),sum(d) from t033t1 group by (a) +select a,cast(null as int),cast(null as int),min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 group by (a) union all -select cast(null as int),cast(null as int),cast(null as int),sum(d) from t033t1 -) x(a,b,c,d) -order by a,b,c; +select cast(null as int),cast(null as int),cast(null as int),min(d),max(d),sum(d),avg(d),count(d),count(*),count(c) from t033t1 +) x(a,b,c,d,e,f,g,h,i,j) +order by a,b,c,d,e,f,g,h,i,j; select b,c,a,sum(d) from t033t1 group by rollup(b,c,a) order by 1,2,3; @@ -61,7 +65,7 @@ select b,cast(null as int),cast(null as int),sum(d) from t033t1 group by (b) union all select cast(null as int),cast(null as int),cast(null as int),sum(d) from t033t1 ) x(a,b,c,d) -order by a,b,c; +order by 1,2,3; select a,b,c from t033t1 group by rollup (a,b,c) order by 1,2,3; @@ -84,10 +88,45 @@ select cast(d as int), sum(a) from t033t1 group by rollup (cast(d as int)); select cast(d as nullable), sum(a) from t033t1 group by rollup (1); select cast(d as nullable), sum(a) from t033t1 group by rollup (cast(d as nullable)); +-- firstN and cancel processing +select [first 1] a,b,sum(c) from t033t1 group by rollup(1,2); +select [first 9] a,b,sum(c) from t033t1 group by rollup(1,2); + +-- order by descending +select a from t033t1 group by rollup(a) order by a desc; + -- having clause select a, sum(b) from t033t1 group by rollup (a) having a is null; select a, sum(b) from t033t1 group by rollup (a) having a is not null; +-- grby rollup and joins and subqueries +prepare s from +select * from (select a,sum(b) from t033t1 group by rollup(a)) x(a1,b1), + (select a,sum(b) from t033t1 group by rollup(a)) y(a2,b2) + where x.a1 = y.a2; +explain options 'f' s; +execute s; + +prepare s from +select * from (select a,sum(b) from t033t1 group by rollup(a)) x(a1,b1), + (select a,sum(b) from t033t1 group by rollup(a)) y(a2,b2) + where x.a1 = y.a2 or (x.a1 is null and y.a2 is null); +explain options 'f' s; +execute s; + +-- should not convert subq to join and eliminate groupby rollup +prepare s from +select a from t033t1 where a in (select a from t033t1 group by rollup(a)) + or a is null order by a; +explain options 'f' s; +execute s; + +prepare s from +select a from t033t1 where a in (select NVL(a, 2) from t033t1 group by rollup(a)) + order by 1; +explain options 'f' s; +execute s; + -- partitioned table with esp execution drop table if exists t033t2 cascade; create table t033t2 (z int not null primary key, From 88f9ad6f80286e79e59e73c50d88a3c3457c3338 Mon Sep 17 00:00:00 2001 From: Anoop Sharma Date: Thu, 29 Sep 2016 13:58:59 +0000 Subject: [PATCH 3/3] fixes to group by rollup change cache key computation changes based on review comment. And another change that only shows up during release run (some old code was within #ifndef DEBUG define) --- core/sql/generator/GenRelGrby.cpp | 2 +- core/sql/optimizer/RelCache.cpp | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/sql/generator/GenRelGrby.cpp b/core/sql/generator/GenRelGrby.cpp index bed0c6f5a1..16b543fc32 100644 --- a/core/sql/generator/GenRelGrby.cpp +++ b/core/sql/generator/GenRelGrby.cpp @@ -351,7 +351,7 @@ short GroupByAgg::genAggrGrbyExpr(Generator * generator, workAtp, workAtpIndex, tupleDesc, - ExpTupleDesc::SHORT_FORMAT); + (isRollup() ? ExpTupleDesc::LONG_FORMAT : ExpTupleDesc::SHORT_FORMAT)); NADELETEBASIC(attrs, generator->wHeap()); diff --git a/core/sql/optimizer/RelCache.cpp b/core/sql/optimizer/RelCache.cpp index 8c1440a09b..ccd5f64d78 100644 --- a/core/sql/optimizer/RelCache.cpp +++ b/core/sql/optimizer/RelCache.cpp @@ -420,7 +420,15 @@ void GroupByAgg::generateCacheKey(CacheWA &cwa) const cwa += " gBy:"; if (isRollup()) - cwa += " roll:"; + { + cwa += " roll:"; + + ItemExpr * ie = rollupGroupExprList().rebuildExprTree(ITM_ITEM_LIST); + if (ie) + { + ie->generateCacheKey(cwa); + } + } grpExpr->generateCacheKey(cwa); }