Skip to content

Commit

Permalink
TEIID-2804 reducing the scope of the empty check and correcting to only
Browse files Browse the repository at this point in the history 
use count(*)
  • Loading branch information
@shawkins
shawkins committed Jan 14, 2014
1 parent b623972 commit 7120940
Show file tree
Hide file tree
Showing 4 changed files with 189 additions and 95 deletions.
84 changes: 34 additions & 50 deletions engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushAggregates.java
View file
Expand Up @@ -149,14 +149,18 @@ public PlanNode execute(PlanNode plan,
if (aggregates.isEmpty() && stagedGroupingSymbols.isEmpty()) {
continue;
}
addGroupBy(child, new ArrayList<Expression>(stagedGroupingSymbols), aggregates, metadata, groupNode.getParent(), capFinder, false, stagedGroupingSymbols.isEmpty());
addGroupBy(child, new ArrayList<Expression>(stagedGroupingSymbols), aggregates, metadata, groupNode.getParent(), capFinder, false, stagedGroupingSymbols.isEmpty() && containsNullDependent(aggregates));
} else if (groupNode.getFirstChild() == access
&& RuleRaiseAccess.canRaiseOverGroupBy(groupNode, child, aggregates, metadata, capFinder, analysisRecord, false)
&& canFilterEmpty(metadata, capFinder, aggregates, child)) {
addEmptyFilter(aggregates, groupNode, metadata, capFinder, RuleRaiseAccess.getModelIDFromAccess(child, metadata));
&& canFilterEmpty(metadata, capFinder, child, groupingExpressions)) {
if (groupingExpressions.isEmpty()) {
addEmptyFilter(aggregates, groupNode, metadata, capFinder, RuleRaiseAccess.getModelIDFromAccess(child, metadata));
}
FrameUtil.convertNode(groupNode.getParent(), null, null, ((SymbolMap)groupNode.getProperty(Info.SYMBOL_MAP)).inserseMapping(), metadata, false);
RuleRaiseAccess.performRaise(null, access, access.getParent());
RuleRaiseAccess.performRaise(null, access, access.getParent());
if (groupingExpressions.isEmpty() && RuleRaiseAccess.canRaiseOverSelect(access, metadata, capFinder, access.getParent(), null)) {
RuleRaiseAccess.performRaise(null, access, access.getParent());
}
}
}
//TODO: consider pushing aggregate in general
Expand Down Expand Up @@ -280,7 +284,7 @@ private void pushGroupNodeOverUnion(QueryMetadataInterface metadata, Capabilitie
Iterator<Boolean> pushdownIterator = pushdownList.iterator();
boolean first = true;
for (PlanNode planNode : unionChildren) {
addUnionGroupBy(groupingExpressions, aggregates, parentMap, metadata, capFinder, group, first, planNode, !pushdownIterator.next());
addUnionGroupBy(groupingExpressions, aggregates, parentMap, metadata, capFinder, group, first, planNode, !pushdownIterator.next(), false);
first = false;
}

Expand Down Expand Up @@ -401,7 +405,7 @@ private void decomposeGroupBy(PlanNode groupNode, PlanNode sourceNode,
for (PlanNode planNode : unionChildren) {
addUnionGroupBy(groupingExpressions, aggregates,
parentMap, metadata, capFinder, group, first,
planNode, false);
planNode, false, true);
first = false;
}
List<Expression> symbols = (List<Expression>) NodeEditor.findNodePreOrder(sourceNode, NodeConstants.Types.PROJECT).getProperty(Info.PROJECT_COLS);
Expand Down Expand Up @@ -466,7 +470,7 @@ private void addUnionGroupBy(
List<Expression> groupingExpressions,
LinkedHashSet<AggregateSymbol> aggregates, SymbolMap parentMap,
QueryMetadataInterface metadata, CapabilitiesFinder capFinder,
GroupSymbol group, boolean first, PlanNode planNode, boolean viewOnly)
GroupSymbol group, boolean first, PlanNode planNode, boolean viewOnly, boolean partitioned)
throws QueryMetadataException, TeiidComponentException,
QueryPlannerException, QueryResolverException {
List<Expression> groupingColumns = LanguageObject.Util.deepClone(groupingExpressions, Expression.class);
Expand Down Expand Up @@ -537,7 +541,7 @@ private void addUnionGroupBy(
view.addAsParent(projectPlanNode);

if (!viewOnly) {
addGroupBy(view, groupingColumns, aggregates, metadata, projectPlanNode.getParent(), capFinder, true, groupingColumns.isEmpty() || containsNullDependent(aggregates));
addGroupBy(view, groupingColumns, aggregates, metadata, projectPlanNode.getParent(), capFinder, true, groupingColumns.isEmpty() && (partitioned || containsNullDependent(aggregates)));
}
}

Expand All @@ -563,17 +567,15 @@ private boolean canPushGroupByToUnionChild(QueryMetadataInterface metadata,
if (!result) {
return false;
}
if ((!groupingExpressions.isEmpty() || containsNullDependent(aggregates)) && !canFilterEmpty(metadata, capFinder, aggregates, planNode)) {
if (containsNullDependent(aggregates) && !canFilterEmpty(metadata, capFinder, planNode, groupingExpressions)) {
return false;
}
return true;
}

private boolean containsNullDependent(Set<AggregateSymbol> aggregates) {
private boolean containsNullDependent(Collection<AggregateSymbol> aggregates) {
for (AggregateSymbol aggregateSymbol : aggregates) {
if (aggregateSymbol.getAggregateFunction() == Type.COUNT) {
return true;
}
//we don't consider count here as we dealing with the original aggregates, not the mapped expressions
if (aggregateSymbol.getFunctionDescriptor() != null && aggregateSymbol.getFunctionDescriptor().isNullDependent()) {
return true;
}
Expand All @@ -582,21 +584,15 @@ private boolean containsNullDependent(Set<AggregateSymbol> aggregates) {
}

private boolean canFilterEmpty(QueryMetadataInterface metadata,
CapabilitiesFinder capFinder, Set<AggregateSymbol> aggregates,
PlanNode planNode) throws QueryMetadataException,
CapabilitiesFinder capFinder, PlanNode planNode, List<Expression> groupingExpressions) throws QueryMetadataException,
TeiidComponentException {
Object modelId = RuleRaiseAccess.getModelIDFromAccess(planNode, metadata);
if (CapabilitiesUtil.supports(Capability.CRITERIA_ISNULL, modelId, metadata, capFinder) && CapabilitiesUtil.supports(Capability.CRITERIA_NOT, modelId, metadata, capFinder)) {
for (AggregateSymbol as : aggregates) {
if (as.getAggregateFunction() != AggregateSymbol.Type.COUNT) {
if (as.getAggregateFunction() == AggregateSymbol.Type.COUNT && as.getFunctionDescriptor().isNullDependent()) {
continue;
}
return true;
}
}
if (!groupingExpressions.isEmpty()) {
return true;
}
return (CapabilitiesUtil.supports(Capability.CRITERIA_COMPARE_ORDERED, modelId, metadata, capFinder) && CapabilitiesUtil.supports(Capability.QUERY_AGGREGATES_COUNT_STAR, modelId, metadata, capFinder));
Object modelId = RuleRaiseAccess.getModelIDFromAccess(planNode, metadata);
return (/*CapabilitiesUtil.supports(Capability.CRITERIA_COMPARE_ORDERED, modelId, metadata, capFinder)
&&*/ CapabilitiesUtil.supports(Capability.QUERY_AGGREGATES_COUNT_STAR, modelId, metadata, capFinder)
/*&& CapabilitiesUtil.supports(Capability.QUERY_HAVING, modelId, metadata, capFinder)*/);
}

/**
Expand Down Expand Up @@ -751,7 +747,7 @@ private void pushGroupNode(PlanNode groupNode,
if (aggregates.isEmpty() && stagedGroupingSymbols.isEmpty()) {
continue;
}
addGroupBy(planNode, new ArrayList<Expression>(stagedGroupingSymbols), aggregates, metadata, groupNode.getParent(), capFinder, true, stagedGroupingSymbols.isEmpty());
addGroupBy(planNode, new ArrayList<Expression>(stagedGroupingSymbols), aggregates, metadata, groupNode.getParent(), capFinder, true, stagedGroupingSymbols.isEmpty() && containsNullDependent(aggregates));
}
}

Expand All @@ -765,11 +761,12 @@ private void addGroupBy(
PlanNode stageGroup = NodeFactory.getNewNode(NodeConstants.Types.GROUP);
child.addAsParent(stageGroup);
aggregates = new LinkedHashSet<AggregateSymbol>(aggregates);
Map<Expression, ElementSymbol> reverseMapping = RelationalPlanner.buildGroupingNode(aggregates, stagedGroupingSymbols, stageGroup, context, idGenerator).inserseMapping();
if (filterEmpty) {
// if the source has no rows we need to insert a select node with criteria
addEmptyFilter(aggregates, stageGroup, metadata, capFinder, RuleRaiseAccess.getModelIDFromAccess(NodeEditor.findNodePreOrder(child, NodeConstants.Types.ACCESS), metadata));
}
Map<Expression, ElementSymbol> reverseMapping = RelationalPlanner.buildGroupingNode(aggregates, stagedGroupingSymbols, stageGroup, context, idGenerator).inserseMapping();

GroupSymbol newGroup = reverseMapping.values().iterator().next().getGroupSymbol();
PlanNode node = stageGroup.getParent();
while (node != endNode) {
Expand All @@ -795,8 +792,8 @@ private void addGroupBy(
accessNode.getGroups().clear();
accessNode.getGroups().addAll(stageGroup.getGroups());
RuleRaiseAccess.performRaise(null, accessNode, stageGroup);
if (filterEmpty) {
RuleRaiseAccess.performRaise(null, stageGroup.getParent(), stageGroup.getParent().getParent());
if (filterEmpty && RuleRaiseAccess.canRaiseOverSelect(accessNode, metadata, capFinder, accessNode.getParent(), null)) {
RuleRaiseAccess.performRaise(null, accessNode, accessNode.getParent());
}
}
}
Expand All @@ -805,25 +802,9 @@ private void addGroupBy(
private void addEmptyFilter(Collection<AggregateSymbol> aggregates,
PlanNode stageGroup, QueryMetadataInterface metadata, CapabilitiesFinder capFinder, Object modelId) throws QueryMetadataException, TeiidComponentException {
PlanNode selectNode = NodeFactory.getNewNode(NodeConstants.Types.SELECT);
Criteria crit = null;
if (CapabilitiesUtil.supports(Capability.CRITERIA_ISNULL, modelId, metadata, capFinder) && CapabilitiesUtil.supports(Capability.CRITERIA_NOT, modelId, metadata, capFinder)) {
for (AggregateSymbol aggregateSymbol : aggregates) {
if (aggregateSymbol.getAggregateFunction() != AggregateSymbol.Type.COUNT) {
if (aggregateSymbol.getAggregateFunction() == AggregateSymbol.Type.COUNT && aggregateSymbol.getFunctionDescriptor().isNullDependent()) {
continue;
}
IsNullCriteria inc = new IsNullCriteria((Expression) aggregateSymbol.clone());
inc.setNegated(true);
crit = inc;
break;
}
}
}
if (crit == null) {
AggregateSymbol count = new AggregateSymbol(NonReserved.COUNT, false, null);
aggregates.add(count); //consider the count aggregate for the push down call below
crit = new CompareCriteria(count, CompareCriteria.GT, new Constant(new Integer(0)));
}
AggregateSymbol count = new AggregateSymbol(NonReserved.COUNT, false, null);
aggregates.add(count); //consider the count aggregate for the push down call below
Criteria crit = new CompareCriteria(count, CompareCriteria.GT, new Constant(new Integer(0)));
selectNode.setProperty(NodeConstants.Info.SELECT_CRITERIA, crit);
selectNode.setProperty(NodeConstants.Info.IS_HAVING, Boolean.TRUE);
stageGroup.addAsParent(selectNode);
Expand Down Expand Up @@ -1086,7 +1067,10 @@ private static Map<AggregateSymbol, Expression> buildAggregateMap(Collection<? e
nestedAggregates.add(sumSqAgg);
} else {
//AGG(X) -> AGG(AGG(X))
newExpression = new AggregateSymbol(aggFunction.name(), false, partitionAgg);
newExpression = new AggregateSymbol(aggFunction.name(), false, partitionAgg);
if (partitionAgg.getFunctionDescriptor() != null) {
((AggregateSymbol)newExpression).setFunctionDescriptor(partitionAgg.getFunctionDescriptor().clone());
}
nestedAggregates.add(partitionAgg);
}

Expand Down
4 changes: 2 additions & 2 deletions ...ava/org/teiid/dqp/internal/process/multisource/TestMultiSourcePlanToProcessConverter.java
View file
Expand Up @@ -314,7 +314,7 @@ public ProcessorPlan helpTestMultiSourcePlan(QueryMetadataInterface metadata, St
Arrays.asList("y", "b"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
};
final HardcodedDataManager dataMgr = new HardcodedDataManager();
dataMgr.addData("SELECT MAX(g_0.a) FROM MultiModel.Phys AS g_0 HAVING MAX(g_0.a) IS NOT NULL", //$NON-NLS-1$
dataMgr.addData("SELECT MAX(g_0.a) FROM MultiModel.Phys AS g_0", //$NON-NLS-1$
new List<?>[] {
Arrays.asList("y")}); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
ProcessorPlan plan = helpTestMultiSourcePlan(metadata, userSql, multiModel, sources, dataMgr, expected, RealMetadataFactory.exampleMultiBindingVDB());
Expand Down Expand Up @@ -346,7 +346,7 @@ public ProcessorPlan helpTestMultiSourcePlan(QueryMetadataInterface metadata, St
Arrays.asList("y"), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
};
final HardcodedDataManager dataMgr = new HardcodedDataManager();
dataMgr.addData("SELECT MAX(g_0.a) FROM MultiModel.Phys AS g_0 HAVING MAX(g_0.a) IS NOT NULL", //$NON-NLS-1$
dataMgr.addData("SELECT MAX(g_0.a) FROM MultiModel.Phys AS g_0", //$NON-NLS-1$
new List<?>[] {
Arrays.asList("y")});
ProcessorPlan plan = helpTestMultiSourcePlan(metadata, userSql, multiModel, sources, dataMgr, expected, RealMetadataFactory.exampleMultiBindingVDB());
Expand Down
38 changes: 5 additions & 33 deletions engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java
View file
Expand Up @@ -70,7 +70,7 @@ public static CapabilitiesFinder getAggregatesFinder() {

String sql = "SELECT a12.intkey AS REGION_NBR, SUM(a11.intnum) AS WJXBFS1 FROM bqt1.smalla AS a11 INNER JOIN bqt2.smalla AS a12 ON a11.stringkey = a12.stringkey WHERE a11.stringkey = 0 GROUP BY a12.intkey"; //$NON-NLS-1$
ProcessorPlan plan = TestOptimizer.helpPlan(sql, RealMetadataFactory.exampleBQTCached(), null, capFinder,
new String[] {"SELECT SUM(a11.intnum) FROM bqt1.smalla AS a11 WHERE a11.stringkey = '0' HAVING SUM(a11.intnum) IS NOT NULL", "SELECT a12.intkey FROM bqt2.smalla AS a12 WHERE a12.stringkey = '0'"}, TestOptimizer.SHOULD_SUCCEED); //$NON-NLS-1$ //$NON-NLS-2$
new String[] {"SELECT SUM(g_0.IntNum) FROM BQT1.SmallA AS g_0 WHERE g_0.StringKey = '0'", "SELECT g_0.IntKey FROM BQT2.SmallA AS g_0 WHERE g_0.StringKey = '0'"}, TestOptimizer.SHOULD_SUCCEED); //$NON-NLS-1$ //$NON-NLS-2$

TestOptimizer.checkNodeTypes(plan, new int[] {
2, // Access
Expand Down Expand Up @@ -940,7 +940,7 @@ null, getAggregatesFinder(),
capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$

ProcessorPlan plan = TestOptimizer.helpPlan("select e1, count(*) filter (where e3) from (select e1, e2, e3 from pm1.g1 union all select e1, e2, e3 from pm1.g2) y group by e1", RealMetadataFactory.example1Cached(), null, capFinder, //$NON-NLS-1$
new String[]{"SELECT g_0.e1, COUNT(*) FILTER(WHERE g_0.e3) FROM pm1.g1 AS g_0 GROUP BY g_0.e1 HAVING COUNT(*) > 0", "SELECT g_0.e1, COUNT(*) FILTER(WHERE g_0.e3) FROM pm1.g2 AS g_0 GROUP BY g_0.e1 HAVING COUNT(*) > 0"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
new String[]{"SELECT g_0.e1, COUNT(*) FILTER(WHERE g_0.e3) FROM pm1.g1 AS g_0 GROUP BY g_0.e1", "SELECT g_0.e1, COUNT(*) FILTER(WHERE g_0.e3) FROM pm1.g2 AS g_0 GROUP BY g_0.e1"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
TestOptimizer.checkNodeTypes(plan, new int[] {
2, // Access
0, // DependentAccess
Expand Down Expand Up @@ -994,7 +994,7 @@ null, getAggregatesFinder(),
capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$

ProcessorPlan plan = TestOptimizer.helpPlan("select max(e2) from (select e1, e2 from pm1.g1 union all select e1, e2 from pm1.g2) z", RealMetadataFactory.example1Cached(), null, capFinder, //$NON-NLS-1$
new String[]{"SELECT MAX(g_0.e2) FROM pm1.g2 AS g_0 HAVING MAX(g_0.e2) IS NOT NULL", "SELECT MAX(g_0.e2) FROM pm1.g1 AS g_0 HAVING MAX(g_0.e2) IS NOT NULL"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
new String[]{"SELECT MAX(g_0.e2) FROM pm1.g2 AS g_0", "SELECT MAX(g_0.e2) FROM pm1.g1 AS g_0"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
TestOptimizer.checkNodeTypes(plan, new int[] {
2, // Access
0, // DependentAccess
Expand Down Expand Up @@ -1049,35 +1049,7 @@ null, getAggregatesFinder(),
capFinder.addCapabilities("pm2", TestOptimizer.getTypicalCapabilities()); //$NON-NLS-1$

ProcessorPlan plan = TestOptimizer.helpPlan("select max(e2), count(*) from (select e1, e2 from pm1.g1 union all select e1, e2 from pm2.g2) z", RealMetadataFactory.example1Cached(), null, capFinder, //$NON-NLS-1$
new String[]{"SELECT MAX(g_0.e2), COUNT(*) FROM pm1.g1 AS g_0 HAVING MAX(g_0.e2) IS NOT NULL", "SELECT g_0.e2 FROM pm2.g2 AS g_0"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
TestOptimizer.checkNodeTypes(plan, new int[] {
2, // Access
0, // DependentAccess
0, // DependentSelect
0, // DependentProject
0, // DupRemove
1, // Grouping
0, // NestedLoopJoinStrategy
0, // MergeJoinStrategy
0, // Null
0, // PlanExecution
1, // Project
0, // Select
0, // Sort
1 // UnionAll
});
}

@Test public void testPushDownOverUnionMixed1() throws Exception {
FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
BasicSourceCapabilities caps = getAggregateCapabilities();
capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$
BasicSourceCapabilities caps1 = getAggregateCapabilities();
caps1.setCapabilitySupport(Capability.QUERY_FROM_INLINE_VIEWS, false);
capFinder.addCapabilities("pm2", caps1); //$NON-NLS-1$

ProcessorPlan plan = TestOptimizer.helpPlan("select max(e2), count(*) from (select e1, e2 from pm1.g1 union all select e1, e2 from pm2.g2) z", RealMetadataFactory.example1Cached(), null, capFinder, //$NON-NLS-1$
new String[]{"SELECT MAX(g_0.e2), COUNT(*) FROM pm2.g2 AS g_0 HAVING MAX(g_0.e2) IS NOT NULL", "SELECT MAX(g_0.e2), COUNT(*) FROM pm1.g1 AS g_0 HAVING MAX(g_0.e2) IS NOT NULL"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
new String[]{"SELECT g_0.e2 FROM pm2.g2 AS g_0", "SELECT MAX(g_0.e2), COUNT(*) FROM pm1.g1 AS g_0"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
TestOptimizer.checkNodeTypes(plan, new int[] {
2, // Access
0, // DependentAccess
Expand Down Expand Up @@ -1220,7 +1192,7 @@ null, new DefaultCapabilitiesFinder(bsc),
capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$

ProcessorPlan plan = TestOptimizer.helpPlan("select count(*) from (select e1, e2, 1 as part from pm1.g1 union all select e1, e2, 2 as part from pm1.g2) z group by e1", RealMetadataFactory.example1Cached(), null, capFinder, //$NON-NLS-1$
new String[]{"SELECT g_0.e1, COUNT(*) FROM pm1.g1 AS g_0 GROUP BY g_0.e1 HAVING COUNT(*) > 0", "SELECT g_0.e1, COUNT(*) FROM pm1.g2 AS g_0 GROUP BY g_0.e1 HAVING COUNT(*) > 0"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
new String[]{"SELECT g_0.e1, COUNT(*) FROM pm1.g1 AS g_0 GROUP BY g_0.e1", "SELECT g_0.e1, COUNT(*) FROM pm1.g2 AS g_0 GROUP BY g_0.e1"}, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$
TestOptimizer.checkNodeTypes(plan, new int[] {
2, // Access
0, // DependentAccess
Expand Down

0 comments on commit 7120940

Please sign in to comment.