diff --git a/engine/src/main/java/org/teiid/query/function/aggregate/JSONArrayAgg.java b/engine/src/main/java/org/teiid/query/function/aggregate/JSONArrayAgg.java index 17cd1af279..f4c975dc00 100644 --- a/engine/src/main/java/org/teiid/query/function/aggregate/JSONArrayAgg.java +++ b/engine/src/main/java/org/teiid/query/function/aggregate/JSONArrayAgg.java @@ -27,7 +27,7 @@ import org.teiid.query.util.CommandContext; /** - * Aggregates XML entries + * Aggregates Json entries */ public class JSONArrayAgg extends SingleArgumentAggregateFunction { diff --git a/engine/src/main/java/org/teiid/query/optimizer/TriggerActionPlanner.java b/engine/src/main/java/org/teiid/query/optimizer/TriggerActionPlanner.java index 39a93733d8..c73f612f40 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/TriggerActionPlanner.java +++ b/engine/src/main/java/org/teiid/query/optimizer/TriggerActionPlanner.java @@ -36,7 +36,7 @@ import org.teiid.query.metadata.QueryMetadataInterface; import org.teiid.query.optimizer.capabilities.CapabilitiesFinder; import org.teiid.query.optimizer.relational.rules.RuleChooseJoinStrategy; -import org.teiid.query.optimizer.relational.rules.RuleMergeCriteria; +import org.teiid.query.optimizer.relational.rules.RulePlanSubqueries; import org.teiid.query.processor.ProcessorPlan; import org.teiid.query.processor.proc.ForEachRowPlan; import org.teiid.query.processor.proc.ProcedurePlan; @@ -171,7 +171,7 @@ private ProcessorPlan rewritePlan(TriggerAction ta, IDGenerator idGenerator, //map to the query form - changes references back to element form SymbolMap queryMapping = new SymbolMap(); queryMapping.asUpdatableMap().putAll(mapping); - ExpressionMappingVisitor visitor = new RuleMergeCriteria.ReferenceReplacementVisitor(queryMapping); + ExpressionMappingVisitor visitor = new RulePlanSubqueries.ReferenceReplacementVisitor(queryMapping); DeepPostOrderNavigator.doVisit(queryExpression.getSelect(), visitor); //now we can return a plan based off a single insert statement @@ -181,7 +181,7 @@ private ProcessorPlan rewritePlan(TriggerAction ta, IDGenerator idGenerator, List values = mapped.getValues(); SymbolMap queryMapping = new SymbolMap(); queryMapping.asUpdatableMap().putAll(mapping); - ExpressionMappingVisitor visitor = new RuleMergeCriteria.ReferenceReplacementVisitor(queryMapping); + ExpressionMappingVisitor visitor = new RulePlanSubqueries.ReferenceReplacementVisitor(queryMapping); Select select = new Select(); select.addSymbols(values); DeepPostOrderNavigator.doVisit(select, visitor); diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java b/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java index 64c17004a3..ec14c02582 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java @@ -991,8 +991,10 @@ public RuleStack buildRules() { //TODO: update plan sorts to take advantage of semi-join ordering if (hints.hasJoin || hints.hasCriteria || hints.hasRowBasedSecurity) { - rules.push(new RuleMergeCriteria(idGenerator, capFinder, analysisRecord, context, metadata)); + rules.push(RuleConstants.MERGE_CRITERIA); } + + rules.push(new RulePlanSubqueries(idGenerator, capFinder, analysisRecord, context, metadata)); if(hints.hasJoin) { rules.push(RuleConstants.IMPLEMENT_JOIN_STRATEGY); @@ -1681,7 +1683,7 @@ void buildTree(FromClause clause, final PlanNode parent) //insert is null criteria IsNullCriteria criteria = new IsNullCriteria((Expression) at.getArrayValue().clone()); if (sfc.getCommand().getCorrelatedReferences() != null) { - RuleMergeCriteria.ReferenceReplacementVisitor rrv = new RuleMergeCriteria.ReferenceReplacementVisitor(sfc.getCommand().getCorrelatedReferences()); + RulePlanSubqueries.ReferenceReplacementVisitor rrv = new RulePlanSubqueries.ReferenceReplacementVisitor(sfc.getCommand().getCorrelatedReferences()); PreOrPostOrderNavigator.doVisit(criteria, rrv, PreOrPostOrderNavigator.PRE_ORDER); } criteria.setNegated(true); @@ -1893,6 +1895,9 @@ public static PlanNode createSelectNode(final Criteria crit, boolean isHaving) { /*|| !ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(crit).isEmpty()*/)) { critNode.setProperty(NodeConstants.Info.IS_HAVING, Boolean.TRUE); } + if (crit instanceof DependentSetCriteria) { + critNode.setProperty(Info.IS_DEPENDENT_SET, true); + } // Add groups to crit node critNode.addGroups(GroupsUsedByElementsVisitor.getGroups(crit)); critNode.addGroups(GroupsUsedByElementsVisitor.getGroups(critNode.getCorrelatedReferenceElements())); diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/plantree/PlanNode.java b/engine/src/main/java/org/teiid/query/optimizer/relational/plantree/PlanNode.java index b35979614c..e7b097ab1c 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/plantree/PlanNode.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/plantree/PlanNode.java @@ -311,8 +311,7 @@ public void replaceChild(PlanNode child, PlanNode replacement) { } /** - * Add the node as this node's parent. NOTE: This node - * must already have a parent. + * Add the node as this node's parent. * @param node */ public void addAsParent(PlanNode node) { diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/NewCalculateCostUtil.java b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/NewCalculateCostUtil.java index 5584602a1a..0bfafd9843 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/NewCalculateCostUtil.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/NewCalculateCostUtil.java @@ -443,7 +443,7 @@ private static void estimateJoinNodeCost(PlanNode node, QueryMetadataInterface m nonEquiJoinCriteria = joinCriteria; } - if (!nonEquiJoinCriteria.isEmpty()) { + if (nonEquiJoinCriteria != null && !nonEquiJoinCriteria.isEmpty()) { Criteria crit = Criteria.combineCriteria(nonEquiJoinCriteria); //TODO: we may be able to get a fairly accurate join estimate if the //unknown side is being joined with a key @@ -1575,7 +1575,10 @@ private static LinkedList determineTargets( throws QueryPlannerException, TeiidComponentException { LinkedList targets = new LinkedList(); LinkedList critNodes = new LinkedList(); - critNodes.add(RelationalPlanner.createSelectNode(new DependentSetCriteria(depExpr, null), false)); + PlanNode select = RelationalPlanner.createSelectNode(new DependentSetCriteria(depExpr, null), false); + // mark as not a dependent set so that the flag doesn't inadvertently change anything + select.setProperty(Info.IS_DEPENDENT_SET, false); + critNodes.add(select); LinkedList initialTargets = new LinkedList(); initialTargets.add(dependentNode); while (!critNodes.isEmpty()) { diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleChooseDependent.java b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleChooseDependent.java index 84550fd67a..d5235e4ecd 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleChooseDependent.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleChooseDependent.java @@ -675,10 +675,7 @@ public static PlanNode getDependentCriteriaNode(String id, List inde static PlanNode createDependentSetNode(String id, List expressions) { DependentSetCriteria crit = createDependentSetCriteria(id, expressions); - PlanNode selectNode = RelationalPlanner.createSelectNode(crit, false); - - selectNode.setProperty(NodeConstants.Info.IS_DEPENDENT_SET, Boolean.TRUE); - return selectNode; + return RelationalPlanner.createSelectNode(crit, false); } static DependentSetCriteria createDependentSetCriteria(String id, List expressions) { diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleCollapseSource.java b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleCollapseSource.java index 00f533e0db..66a004db07 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleCollapseSource.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleCollapseSource.java @@ -707,7 +707,7 @@ void buildQuery(PlanNode accessRoot, PlanNode node, Query query, CommandContext SymbolMap map = (SymbolMap)node.getProperty(NodeConstants.Info.CORRELATED_REFERENCES); if (map != null) { - ExpressionMappingVisitor visitor = new RuleMergeCriteria.ReferenceReplacementVisitor(map); + ExpressionMappingVisitor visitor = new RulePlanSubqueries.ReferenceReplacementVisitor(map); DeepPostOrderNavigator.doVisit(newQuery, visitor); sfc.setLateral(true); } @@ -883,7 +883,7 @@ public static void prepareSubquery(SubqueryContainer container) { } final SymbolMap map = container.getCommand().getCorrelatedReferences(); if (map != null) { - ExpressionMappingVisitor visitor = new RuleMergeCriteria.ReferenceReplacementVisitor(map); + ExpressionMappingVisitor visitor = new RulePlanSubqueries.ReferenceReplacementVisitor(map); DeepPostOrderNavigator.doVisit(command, visitor); } command.setProcessorPlan(container.getCommand().getProcessorPlan()); diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleConstants.java b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleConstants.java index c6009624df..f167aa32a5 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleConstants.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleConstants.java @@ -49,4 +49,5 @@ private RuleConstants() { } public static final OptimizerRule SUBSTITUTE_EXPRESSIONS = new RuleSubstituteExpressions(); public static final OptimizerRule PLAN_OUTER_JOINS = new RulePlanOuterJoins(); public static final OptimizerRule PUSH_LARGE_IN = new RulePushLargeIn(); + public static final OptimizerRule MERGE_CRITERIA = new RuleMergeCriteria(); } diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeCriteria.java b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeCriteria.java index bcde01209f..b6ba179c00 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeCriteria.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeCriteria.java @@ -19,148 +19,40 @@ package org.teiid.query.optimizer.relational.rules; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Set; -import org.teiid.api.exception.query.QueryMetadataException; import org.teiid.api.exception.query.QueryPlannerException; -import org.teiid.client.plan.Annotation; -import org.teiid.client.plan.Annotation.Priority; import org.teiid.core.TeiidComponentException; -import org.teiid.core.id.IDGenerator; import org.teiid.query.analysis.AnalysisRecord; import org.teiid.query.metadata.QueryMetadataInterface; -import org.teiid.query.metadata.SupportConstants; -import org.teiid.query.optimizer.QueryOptimizer; import org.teiid.query.optimizer.capabilities.CapabilitiesFinder; import org.teiid.query.optimizer.relational.OptimizerRule; import org.teiid.query.optimizer.relational.RuleStack; import org.teiid.query.optimizer.relational.plantree.NodeConstants; import org.teiid.query.optimizer.relational.plantree.NodeEditor; -import org.teiid.query.optimizer.relational.plantree.NodeFactory; import org.teiid.query.optimizer.relational.plantree.PlanNode; -import org.teiid.query.processor.relational.JoinNode.JoinStrategyType; -import org.teiid.query.processor.relational.MergeJoinStrategy.SortOption; -import org.teiid.query.processor.relational.RelationalPlan; -import org.teiid.query.resolver.util.ResolverUtil; import org.teiid.query.rewriter.QueryRewriter; -import org.teiid.query.sql.LanguageObject; -import org.teiid.query.sql.lang.*; -import org.teiid.query.sql.navigator.DeepPostOrderNavigator; -import org.teiid.query.sql.symbol.AggregateSymbol; -import org.teiid.query.sql.symbol.AggregateSymbol.Type; -import org.teiid.query.sql.symbol.Constant; -import org.teiid.query.sql.symbol.ElementSymbol; -import org.teiid.query.sql.symbol.Expression; -import org.teiid.query.sql.symbol.GroupSymbol; -import org.teiid.query.sql.symbol.Reference; -import org.teiid.query.sql.symbol.ScalarSubquery; -import org.teiid.query.sql.util.SymbolMap; -import org.teiid.query.sql.visitor.AggregateSymbolCollectorVisitor; -import org.teiid.query.sql.visitor.CommandCollectorVisitor; -import org.teiid.query.sql.visitor.ExpressionMappingVisitor; +import org.teiid.query.sql.lang.CompoundCriteria; +import org.teiid.query.sql.lang.Criteria; import org.teiid.query.sql.visitor.GroupsUsedByElementsVisitor; -import org.teiid.query.sql.visitor.ReferenceCollectorVisitor; import org.teiid.query.util.CommandContext; public final class RuleMergeCriteria implements OptimizerRule { - /** - * Used to replace correlated references - */ - public static final class ReferenceReplacementVisitor extends - ExpressionMappingVisitor { - private final SymbolMap refs; - private boolean replacedAny; - - public ReferenceReplacementVisitor(SymbolMap refs) { - super(null); - this.refs = refs; - } - - public Expression replaceExpression(Expression element) { - if (element instanceof Reference) { - Reference r = (Reference)element; - Expression ex = refs.getMappedExpression(r.getExpression()); - if (ex != null) { - if (ex instanceof ElementSymbol) { - ElementSymbol es = (ElementSymbol) ex.clone(); - es.setIsExternalReference(false); - ex = es; - } - replacedAny = true; - return ex; - } - } - return element; - } - - } - - public static class PlannedResult { - public List leftExpressions = new LinkedList(); - public List rightExpressions = new LinkedList(); - public Query query; - public boolean not; - public List nonEquiJoinCriteria = new LinkedList(); - public Criteria additionalCritieria; - public Class type; - public boolean mergeJoin; - public boolean madeDistinct; - public boolean makeInd; - public void reset() { - this.leftExpressions.clear(); - this.rightExpressions.clear(); - this.query = null; - this.not = false; - this.nonEquiJoinCriteria.clear(); - this.additionalCritieria = null; - this.type = null; - this.mergeJoin = false; - this.madeDistinct = false; - this.makeInd = false; - } - } - - private IDGenerator idGenerator; - private CapabilitiesFinder capFinder; - private AnalysisRecord analysisRecord; - private CommandContext context; - private QueryMetadataInterface metadata; - private boolean dependent; - - public RuleMergeCriteria(IDGenerator idGenerator, CapabilitiesFinder capFinder, AnalysisRecord analysisRecord, CommandContext context, QueryMetadataInterface metadata) { - this.idGenerator = idGenerator; - this.capFinder = capFinder; - this.analysisRecord = analysisRecord; - this.context = context; - this.metadata = metadata; - } - /** * @see OptimizerRule#execute(PlanNode, QueryMetadataInterface, RuleStack) */ public PlanNode execute(PlanNode plan, QueryMetadataInterface metadata, CapabilitiesFinder capFinder, RuleStack rules, AnalysisRecord analysisRecord, CommandContext context) throws QueryPlannerException, TeiidComponentException { - dependent = false; // Find strings of criteria and merge them, removing duplicates List criteriaChains = new ArrayList(); - findCriteriaChains(plan, criteriaChains, analysisRecord); + findCriteriaChains(plan, criteriaChains); // Merge chains for (PlanNode critNode : criteriaChains) { mergeChain(critNode, metadata); } - if (dependent) { - //rules.push(new RuleAssignOutputElements(true)); - rules.push(RuleConstants.PUSH_SELECT_CRITERIA); - } return plan; } @@ -169,7 +61,7 @@ public PlanNode execute(PlanNode plan, QueryMetadataInterface metadata, Capabili * @param node Root node to search * @param foundNodes Roots of criteria chains */ - void findCriteriaChains(PlanNode root, List foundNodes, AnalysisRecord analysisRecord) + void findCriteriaChains(PlanNode root, List foundNodes) throws QueryPlannerException, TeiidComponentException { PlanNode recurseRoot = root; @@ -177,15 +69,7 @@ void findCriteriaChains(PlanNode root, List foundNodes, AnalysisRecord // Walk to end of the chain and change recurse root while(recurseRoot.getType() == NodeConstants.Types.SELECT) { - // Look for opportunities to replace with a semi-join - recurseRoot = planMergeJoin(recurseRoot, root); - if (root.getChildCount() == 0) { - root = recurseRoot.getFirstChild(); - if (root.getType() != NodeConstants.Types.SELECT) { - root = root.getParent(); - } - } - recurseRoot = recurseRoot.getFirstChild(); + recurseRoot = recurseRoot.getLastChild(); } // Ignore trivial 1-node case @@ -197,7 +81,7 @@ void findCriteriaChains(PlanNode root, List foundNodes, AnalysisRecord if (recurseRoot.getType() != NodeConstants.Types.ACCESS) { for (PlanNode child : recurseRoot.getChildren()) { - findCriteriaChains(child, foundNodes, analysisRecord); + findCriteriaChains(child, foundNodes); } } } @@ -244,511 +128,6 @@ static void mergeChain(PlanNode chainRoot, QueryMetadataInterface metadata) { chainRoot.addGroups(GroupsUsedByElementsVisitor.getGroups(chainRoot.getCorrelatedReferenceElements())); } - /** - * Look for: - * [NOT] EXISTS ( ) - * IN ( ) / SOME ( ) - * - * and replace with a semi join - */ - private PlanNode planMergeJoin(PlanNode current, PlanNode root) throws QueryMetadataException, - TeiidComponentException { - float sourceCost = NewCalculateCostUtil.computeCostForTree(current.getFirstChild(), metadata); - Criteria crit = (Criteria)current.getProperty(NodeConstants.Info.SELECT_CRITERIA); - - PlannedResult plannedResult = findSubquery(crit, true); - if (plannedResult.query == null) { - return current; - } - if (sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE - && sourceCost < RuleChooseDependent.DEFAULT_INDEPENDENT_CARDINALITY && !plannedResult.mergeJoin) { - //TODO: see if a dependent join applies the other direction - return current; - } - - RelationalPlan originalPlan = (RelationalPlan)plannedResult.query.getProcessorPlan(); - Number originalCardinality = originalPlan.getRootNode().getEstimateNodeCardinality(); - if (!plannedResult.mergeJoin && originalCardinality.floatValue() == NewCalculateCostUtil.UNKNOWN_VALUE) { - //TODO: this check isn't really accurate - exists and scalarsubqueries will always have cardinality 2/1 - //if it's currently unknown, removing criteria won't make it any better - return current; - } - - Collection leftGroups = FrameUtil.findJoinSourceNode(current).getGroups(); - - if (!planQuery(leftGroups, false, plannedResult)) { - if (plannedResult.mergeJoin && analysisRecord != null && analysisRecord.recordAnnotations()) { - this.analysisRecord.addAnnotation(new Annotation(Annotation.HINTS, "Could not plan as a merge join: " + crit, "ignoring MJ hint", Priority.HIGH)); //$NON-NLS-1$ //$NON-NLS-2$ - } - return current; - } - - //check if the child is already ordered. TODO: see if the ordering is compatible. - PlanNode childSort = NodeEditor.findNodePreOrder(root, NodeConstants.Types.SORT, NodeConstants.Types.SOURCE | NodeConstants.Types.JOIN); - if (childSort != null) { - if (plannedResult.mergeJoin && analysisRecord != null && analysisRecord.recordAnnotations()) { - this.analysisRecord.addAnnotation(new Annotation(Annotation.HINTS, "Could not plan as a merge join since the parent join requires a sort: " + crit, "ignoring MJ hint", Priority.HIGH)); //$NON-NLS-1$ //$NON-NLS-2$ - } - return current; - } - - //add an order by, which hopefully will get pushed down - plannedResult.query.setOrderBy(new OrderBy(plannedResult.rightExpressions).clone()); - for (OrderByItem item : plannedResult.query.getOrderBy().getOrderByItems()) { - int index = plannedResult.query.getProjectedSymbols().indexOf(item.getSymbol()); - if (index >= 0 && !(item.getSymbol() instanceof ElementSymbol)) { - item.setSymbol((Expression) plannedResult.query.getProjectedSymbols().get(index).clone()); - } - item.setExpressionPosition(index); - } - - try { - //clone the symbols as they may change during planning - List projectedSymbols = LanguageObject.Util.deepClone(plannedResult.query.getProjectedSymbols(), Expression.class); - //NOTE: we could tap into the relationalplanner at a lower level to get this in a plan node form, - //the major benefit would be to reuse the dependent join planning logic if possible. - RelationalPlan subPlan = (RelationalPlan)QueryOptimizer.optimizePlan(plannedResult.query, metadata, idGenerator, capFinder, analysisRecord, context); - Number planCardinality = subPlan.getRootNode().getEstimateNodeCardinality(); - - if (!plannedResult.mergeJoin) { - //if we don't have a specific hint, then use costing - if (planCardinality.floatValue() == NewCalculateCostUtil.UNKNOWN_VALUE - || planCardinality.floatValue() > 10000000 - || (sourceCost == NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() > 1000) - || (sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && sourceCost * originalCardinality.floatValue() < planCardinality.floatValue() / (100 * Math.log(Math.max(4, sourceCost))))) { - //bail-out if both are unknown or the new plan is too large - if (analysisRecord != null && analysisRecord.recordDebug()) { - current.recordDebugAnnotation("cost of merge join plan was not favorable", null, "semi merge join will not be used", analysisRecord, metadata); //$NON-NLS-1$ //$NON-NLS-2$ - } - return current; - } - } - - //assume dependent - if ((sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() != NewCalculateCostUtil.UNKNOWN_VALUE - && planCardinality.floatValue() < sourceCost / 8) || (sourceCost == NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() <= 1000)) { - plannedResult.makeInd = true; - } - - /*if (plannedResult.makeInd - && plannedResult.query.getCorrelatedReferences() == null - && !plannedResult.not - && plannedResult.leftExpressions.size() == 1) { - //TODO: this should just be a dependent criteria node to avoid sorts - }*/ - - current.recordDebugAnnotation("Conditions met (hint or cost)", null, "Converting to a semi merge join", analysisRecord, metadata); //$NON-NLS-1$ //$NON-NLS-2$ - - PlanNode semiJoin = NodeFactory.getNewNode(NodeConstants.Types.JOIN); - semiJoin.addGroups(current.getGroups()); - Set groups = GroupsUsedByElementsVisitor.getGroups(plannedResult.rightExpressions); - semiJoin.addGroups(groups); - semiJoin.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.MERGE); - semiJoin.setProperty(NodeConstants.Info.JOIN_TYPE, plannedResult.not?JoinType.JOIN_ANTI_SEMI:JoinType.JOIN_SEMI); - semiJoin.setProperty(NodeConstants.Info.NON_EQUI_JOIN_CRITERIA, plannedResult.nonEquiJoinCriteria); - List joinCriteria = new ArrayList(); - joinCriteria.addAll(plannedResult.nonEquiJoinCriteria); - for (int i = 0; i < plannedResult.leftExpressions.size(); i++) { - joinCriteria.add(new CompareCriteria((Expression)plannedResult.rightExpressions.get(i), CompareCriteria.EQ, (Expression)plannedResult.leftExpressions.get(i))); - } - semiJoin.setProperty(NodeConstants.Info.JOIN_CRITERIA, joinCriteria); - //nested subqueries are possibly being promoted, so they need their references updated - List refMaps = semiJoin.getAllReferences(); - SymbolMap parentRefs = plannedResult.query.getCorrelatedReferences(); - for (SymbolMap refs : refMaps) { - for (Map.Entry ref : refs.asUpdatableMap().entrySet()) { - Expression expr = ref.getValue(); - if (expr instanceof ElementSymbol) { - Expression convertedExpr = parentRefs.getMappedExpression((ElementSymbol)expr); - if (convertedExpr != null) { - ref.setValue(convertedExpr); - } - } - semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(ref.getValue())); - } - } - semiJoin.setProperty(NodeConstants.Info.LEFT_EXPRESSIONS, plannedResult.leftExpressions); - semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(plannedResult.leftExpressions)); - semiJoin.setProperty(NodeConstants.Info.RIGHT_EXPRESSIONS, plannedResult.rightExpressions); - semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(plannedResult.rightExpressions)); - semiJoin.setProperty(NodeConstants.Info.SORT_RIGHT, SortOption.ALREADY_SORTED); - semiJoin.setProperty(NodeConstants.Info.OUTPUT_COLS, root.getProperty(NodeConstants.Info.OUTPUT_COLS)); - - List childOutput = (List)current.getFirstChild().getProperty(NodeConstants.Info.OUTPUT_COLS); - PlanNode toCorrect = root; - while (toCorrect != current) { - toCorrect.setProperty(NodeConstants.Info.OUTPUT_COLS, childOutput); - toCorrect = toCorrect.getFirstChild(); - } - - PlanNode node = NodeFactory.getNewNode(NodeConstants.Types.ACCESS); - node.setProperty(NodeConstants.Info.PROCESSOR_PLAN, subPlan); - node.setProperty(NodeConstants.Info.OUTPUT_COLS, projectedSymbols); - node.setProperty(NodeConstants.Info.EST_CARDINALITY, planCardinality); - node.addGroups(groups); - root.addAsParent(semiJoin); - semiJoin.addLastChild(node); - PlanNode result = current.getParent(); - NodeEditor.removeChildNode(result, current); - RuleImplementJoinStrategy.insertSort(semiJoin.getFirstChild(), (List) plannedResult.leftExpressions, semiJoin, metadata, capFinder, true, context); - if (plannedResult.makeInd && !plannedResult.not) { - //TODO: would like for an enhanced sort merge with the semi dep option to avoid the sorting - //this is a little different than a typical dependent join in that the right is the independent side - String id = RuleChooseDependent.nextId(); - PlanNode dep = RuleChooseDependent.getDependentCriteriaNode(id, plannedResult.rightExpressions, plannedResult.leftExpressions, node, metadata, null, false, null); - semiJoin.getFirstChild().addAsParent(dep); - semiJoin.setProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE, id); - this.dependent = true; - } - return result; - } catch (QueryPlannerException e) { - //can't be done - probably access patterns - what about dependent - return current; - } - } - - public PlannedResult findSubquery(Expression expr, boolean unnest, PlannedResult result) { - if (expr instanceof ScalarSubquery) { - ScalarSubquery scc = (ScalarSubquery)expr; - if (scc.getSubqueryHint().isNoUnnest()) { - return result; - } - - Query query = (Query)scc.getCommand(); - - if (!isSingleRow(query)) { - return result; - } - - result.type = scc.getClass(); - result.mergeJoin = scc.getSubqueryHint().isMergeJoin(); - if (!unnest && !result.mergeJoin) { - return result; - } - result.makeInd = scc.getSubqueryHint().isDepJoin(); - result.query = query; - } - return result; - } - - private boolean isSingleRow(Query query) { - if (query.hasAggregates() && query.getGroupBy() == null) { - return true; - } - return false; - //todo: unique key - } - - public PlannedResult findSubquery(Criteria crit, boolean unnest) throws TeiidComponentException, QueryMetadataException { - PlannedResult result = new PlannedResult(); - if (crit instanceof SubquerySetCriteria) { - //convert to the quantified form - SubquerySetCriteria ssc = (SubquerySetCriteria)crit; - if (ssc.getSubqueryHint().isNoUnnest()) { - return result; - } - result.not = ssc.isNegated(); - result.type = ssc.getClass(); - crit = new SubqueryCompareCriteria(ssc.getExpression(), ssc.getCommand(), SubqueryCompareCriteria.EQ, SubqueryCompareCriteria.SOME); - ((SubqueryCompareCriteria)crit).setSubqueryHint(ssc.getSubqueryHint()); - } else if (crit instanceof CompareCriteria) { - CompareCriteria cc = (CompareCriteria)crit; - if (cc.getRightExpression() instanceof ScalarSubquery) { - ScalarSubquery ss = (ScalarSubquery)cc.getRightExpression(); - if (ss.getSubqueryHint().isNoUnnest()) { - return result; - } - result.type = ss.getClass(); - //we can only use a semi-join if we know that 1 row will be present - if (ss.getCommand() instanceof Query) { - Query query = (Query)ss.getCommand(); - if (query.getGroupBy() == null && query.hasAggregates()) { - crit = new SubqueryCompareCriteria(cc.getLeftExpression(), ss.getCommand(), cc.getOperator(), SubqueryCompareCriteria.SOME); - ((SubqueryCompareCriteria)crit).setSubqueryHint(ss.getSubqueryHint()); - } - } - } else if (cc.getLeftExpression() instanceof ScalarSubquery) { - //TODO: handle the left case - } - } - if (crit instanceof SubqueryCompareCriteria) { - SubqueryCompareCriteria scc = (SubqueryCompareCriteria)crit; - if (scc.getSubqueryHint().isNoUnnest()) { - return result; - } - if (scc.getPredicateQuantifier() != SubqueryCompareCriteria.SOME - //TODO: could add an inline view if not a query - || !(scc.getCommand() instanceof Query)) { - return result; - } - - Query query = (Query)scc.getCommand(); - Expression rightExpr = SymbolMap.getExpression(query.getProjectedSymbols().get(0)); - - if (result.not && !isNonNull(query, rightExpr)) { - return result; - } - if (result.type == null) { - result.type = scc.getClass(); - } - result.mergeJoin = scc.getSubqueryHint().isMergeJoin(); - if (!unnest && !result.mergeJoin) { - return result; - } - result.makeInd = scc.getSubqueryHint().isDepJoin(); - result.query = query; - result.additionalCritieria = (Criteria)new CompareCriteria(scc.getLeftExpression(), scc.getOperator(), rightExpr).clone(); - } - if (crit instanceof ExistsCriteria) { - ExistsCriteria exists = (ExistsCriteria)crit; - if (exists.getSubqueryHint().isNoUnnest()) { - return result; - } - if (!(exists.getCommand() instanceof Query)) { - return result; - } - result.type = crit.getClass(); - result.not = exists.isNegated(); - //the correlations can only be in where (if no group by or aggregates) or having - result.mergeJoin = exists.getSubqueryHint().isMergeJoin(); - result.makeInd = exists.getSubqueryHint().isDepJoin(); - if (!unnest && !result.mergeJoin) { - return result; - } - result.query = (Query)exists.getCommand(); - } - return result; - } - - private boolean isNonNull(Query query, Expression rightExpr) - throws TeiidComponentException, QueryMetadataException { - if (rightExpr instanceof ElementSymbol) { - ElementSymbol es = (ElementSymbol)rightExpr; - if (metadata.elementSupports(es.getMetadataID(), SupportConstants.Element.NULL)) { - return false; - } - if (!isSimpleJoin(query)) { - return false; - } - } else if (rightExpr instanceof Constant) { - if (((Constant)rightExpr).isNull()) { - return false; - } - } else if (rightExpr instanceof AggregateSymbol) { - AggregateSymbol as = (AggregateSymbol)rightExpr; - if (!as.isCount()) { - return false; - } - } else { - return false; - } - return true; - } - - private boolean isSimpleJoin(Query query) { - if (query.getFrom() != null) { - for (FromClause clause : query.getFrom().getClauses()) { - if (RuleCollapseSource.hasOuterJoins(clause)) { - return false; - } - } - } - return true; - } - - public boolean planQuery(Collection leftGroups, boolean requireDistinct, PlannedResult plannedResult) throws QueryMetadataException, TeiidComponentException { - if ((plannedResult.query.getLimit() != null && !plannedResult.query.getLimit().isImplicit()) || plannedResult.query.getFrom() == null) { - return false; - } - - if ((plannedResult.type == ExistsCriteria.class || plannedResult.type == ScalarSubquery.class) && plannedResult.query.getCorrelatedReferences() == null) { - //we can't really improve on this case - //TODO: do this check earlier - return false; - } - - plannedResult.query = (Query)plannedResult.query.clone(); - for (Command c : CommandCollectorVisitor.getCommands(plannedResult.query)) { - //subqueries either need to be re-resolved or replanned to maintain - //multilevel correlated references. it's easier for now to replan - c.setProcessorPlan(null); - } - plannedResult.query.setLimit(null); - - List rightGroups = plannedResult.query.getFrom().getGroups(); - Set requiredExpressions = new LinkedHashSet(); - final SymbolMap refs = plannedResult.query.getCorrelatedReferences(); - boolean addGroupBy = false; - if (refs != null) { - boolean hasAggregates = plannedResult.query.hasAggregates(); - Criteria where = plannedResult.query.getCriteria(); - if (plannedResult.query.getGroupBy() == null) { - plannedResult.query.setCriteria(null); - } - Criteria having = plannedResult.query.getHaving(); - plannedResult.query.setHaving(null); - if (hasCorrelatedReferences(plannedResult.query, refs)) { - return false; - } - if (plannedResult.query.getGroupBy() == null) { - processCriteria(leftGroups, plannedResult, rightGroups, requiredExpressions, refs, where, null, true); - if (hasAggregates) { - if (!plannedResult.nonEquiJoinCriteria.isEmpty()) { - return false; - } - addGroupBy = true; - if (!canAddGrouping(plannedResult.query.getSelect())) { - return false; - } - if (!canAddGrouping(having)) { - boolean okToAdd = false; - for (Expression ex : (List)plannedResult.rightExpressions) { - if (canAddGrouping(ex)) { - okToAdd = true; - } - } - if (!okToAdd) { - return false; - } - } - } - } - processCriteria(leftGroups, plannedResult, rightGroups, requiredExpressions, refs, having, plannedResult.query.getGroupBy(), false); - } - - if (plannedResult.additionalCritieria != null) { - RuleChooseJoinStrategy.separateCriteria(leftGroups, rightGroups, plannedResult.leftExpressions, plannedResult.rightExpressions, Criteria.separateCriteriaByAnd(plannedResult.additionalCritieria), plannedResult.nonEquiJoinCriteria); - } - - if (plannedResult.leftExpressions.isEmpty()) { - return false; - } - - plannedResult.leftExpressions = RuleChooseJoinStrategy.createExpressionSymbols(plannedResult.leftExpressions); - plannedResult.rightExpressions = RuleChooseJoinStrategy.createExpressionSymbols(plannedResult.rightExpressions); - - if (requireDistinct && !addGroupBy) { - //ensure that uniqueness applies to the in condition - if (plannedResult.rightExpressions.size() > 1 - && (plannedResult.type != SubquerySetCriteria.class || !isDistinct(plannedResult.query, plannedResult.rightExpressions.subList(plannedResult.rightExpressions.size() - 1, plannedResult.rightExpressions.size()), metadata))) { - return false; - } - - if (!isDistinct(plannedResult.query, plannedResult.rightExpressions, metadata)) { - if (plannedResult.type == ExistsCriteria.class) { - if (requiredExpressions.size() > plannedResult.leftExpressions.size()) { - return false; //not an equi join - } - } else if (!requiredExpressions.isEmpty() && !isDistinct(plannedResult.query, plannedResult.query.getProjectedSymbols(), metadata)) { - return false; - } - plannedResult.query.getSelect().setDistinct(true); - plannedResult.madeDistinct = true; - } - } - - //it doesn't matter what the select columns are - if (plannedResult.type == ExistsCriteria.class) { - plannedResult.query.getSelect().clearSymbols(); - } - - if (addGroupBy) { - LinkedHashSet groupingSymbols = new LinkedHashSet(); - for (Expression expr : (List)plannedResult.rightExpressions) { - AggregateSymbolCollectorVisitor.getAggregates(expr, null, groupingSymbols, null, null, null); - } - if (!groupingSymbols.isEmpty()) { - plannedResult.query.setGroupBy((GroupBy) new GroupBy(new ArrayList(groupingSymbols)).clone()); - } - } - HashSet projectedSymbols = new HashSet(); - for (Expression ses : plannedResult.query.getProjectedSymbols()) { - projectedSymbols.add(SymbolMap.getExpression(ses)); - } - for (Expression ses : requiredExpressions) { - if (projectedSymbols.add(ses)) { - plannedResult.query.getSelect().addSymbol((Expression)ses.clone()); - } - } - for (Expression ses : (List)plannedResult.rightExpressions) { - if (projectedSymbols.add(SymbolMap.getExpression(ses))) { - plannedResult.query.getSelect().addSymbol((Expression)ses.clone()); - } - } - return true; - } - - private boolean canAddGrouping(LanguageObject lo) { - for (AggregateSymbol as : AggregateSymbolCollectorVisitor.getAggregates(lo, false)) { - if (as.isCount() || as.getAggregateFunction() == Type.TEXTAGG) { - //these can be non-null for no rows. - //TODO: could include udaf as well - return false; - } - } - return true; - } - - private void processCriteria(Collection leftGroups, - PlannedResult plannedResult, List rightGroups, - Set requiredExpressions, final SymbolMap refs, - Criteria joinCriteria, GroupBy groupBy, boolean where) { - if (joinCriteria == null) { - return; - } - List crits = Criteria.separateCriteriaByAnd((Criteria)joinCriteria.clone()); - - for (Iterator critIter = crits.iterator(); critIter.hasNext();) { - Criteria conjunct = critIter.next(); - List additionalRequired = new LinkedList(); - AggregateSymbolCollectorVisitor.getAggregates(conjunct, additionalRequired, additionalRequired, additionalRequired, null, groupBy!=null?groupBy.getSymbols():null); - ReferenceReplacementVisitor emv = new ReferenceReplacementVisitor(refs); - DeepPostOrderNavigator.doVisit(conjunct, emv); - if (!emv.replacedAny) { - //if not correlated, then leave it on the query - critIter.remove(); - if (where) { - plannedResult.query.setCriteria(Criteria.combineCriteria(plannedResult.query.getCriteria(), conjunct)); - } else { - plannedResult.query.setHaving(Criteria.combineCriteria(plannedResult.query.getHaving(), conjunct)); - } - } else { - requiredExpressions.addAll(additionalRequired); - } - } - RuleChooseJoinStrategy.separateCriteria(leftGroups, rightGroups, plannedResult.leftExpressions, plannedResult.rightExpressions, crits, plannedResult.nonEquiJoinCriteria); - } - - public static boolean isDistinct(Query query, List expressions, QueryMetadataInterface metadata) - throws QueryMetadataException, TeiidComponentException { - boolean distinct = false; - if (query.getGroupBy() != null) { - distinct = true; - for (Expression groupByExpr : query.getGroupBy().getSymbols()) { - if (!expressions.contains(groupByExpr)) { - distinct = false; - break; - } - } - } - if (distinct) { - return true; - } - HashSet keyPreservingGroups = new HashSet(); - ResolverUtil.findKeyPreserved(query, keyPreservingGroups, metadata); - return NewCalculateCostUtil.usesKey(expressions, keyPreservingGroups, metadata, true); - } - - private boolean hasCorrelatedReferences(LanguageObject object, SymbolMap correlatedReferences) { - Collection references = ReferenceCollectorVisitor.getReferences(object); - for (Reference reference : references) { - if (correlatedReferences.asMap().containsKey(reference.getExpression())) { - return true; - } - } - return false; - } - public String toString() { return "MergeCriteria"; //$NON-NLS-1$ } diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeVirtual.java b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeVirtual.java index 2b3e282c6a..b4dc10e18c 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeVirtual.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeVirtual.java @@ -218,7 +218,7 @@ static PlanNode doMerge(PlanNode frame, groups = FrameUtil.findJoinSourceNode(projectNode).getGroups(); } else if (references != null) { //convert from correlated form to regular references - RuleMergeCriteria.ReferenceReplacementVisitor rrv = new RuleMergeCriteria.ReferenceReplacementVisitor(references); + RulePlanSubqueries.ReferenceReplacementVisitor rrv = new RulePlanSubqueries.ReferenceReplacementVisitor(references); for (Map.Entry entry : symbolMap.asUpdatableMap().entrySet()) { if (entry.getValue() instanceof Reference) { Expression ex = rrv.replaceExpression(entry.getValue()); diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePlanSubqueries.java b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePlanSubqueries.java new file mode 100644 index 0000000000..33505db681 --- /dev/null +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePlanSubqueries.java @@ -0,0 +1,871 @@ +/* + * Copyright Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags and + * the COPYRIGHT.txt file distributed with this work. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.teiid.query.optimizer.relational.rules; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.teiid.api.exception.query.QueryMetadataException; +import org.teiid.api.exception.query.QueryPlannerException; +import org.teiid.client.plan.Annotation; +import org.teiid.client.plan.Annotation.Priority; +import org.teiid.core.TeiidComponentException; +import org.teiid.core.id.IDGenerator; +import org.teiid.logging.LogConstants; +import org.teiid.logging.LogManager; +import org.teiid.query.analysis.AnalysisRecord; +import org.teiid.query.metadata.QueryMetadataInterface; +import org.teiid.query.metadata.SupportConstants; +import org.teiid.query.metadata.TempMetadataAdapter; +import org.teiid.query.metadata.TempMetadataStore; +import org.teiid.query.optimizer.QueryOptimizer; +import org.teiid.query.optimizer.capabilities.CapabilitiesFinder; +import org.teiid.query.optimizer.relational.OptimizerRule; +import org.teiid.query.optimizer.relational.RuleStack; +import org.teiid.query.optimizer.relational.plantree.NodeConstants; +import org.teiid.query.optimizer.relational.plantree.NodeConstants.Info; +import org.teiid.query.optimizer.relational.plantree.NodeEditor; +import org.teiid.query.optimizer.relational.plantree.NodeFactory; +import org.teiid.query.optimizer.relational.plantree.PlanNode; +import org.teiid.query.processor.relational.DependentAccessNode; +import org.teiid.query.processor.relational.JoinNode.JoinStrategyType; +import org.teiid.query.processor.relational.MergeJoinStrategy.SortOption; +import org.teiid.query.processor.relational.RelationalNode; +import org.teiid.query.processor.relational.RelationalPlan; +import org.teiid.query.resolver.util.ResolverUtil; +import org.teiid.query.rewriter.QueryRewriter; +import org.teiid.query.sql.LanguageObject; +import org.teiid.query.sql.lang.*; +import org.teiid.query.sql.navigator.DeepPostOrderNavigator; +import org.teiid.query.sql.symbol.AggregateSymbol; +import org.teiid.query.sql.symbol.AggregateSymbol.Type; +import org.teiid.query.sql.symbol.Constant; +import org.teiid.query.sql.symbol.ElementSymbol; +import org.teiid.query.sql.symbol.Expression; +import org.teiid.query.sql.symbol.GroupSymbol; +import org.teiid.query.sql.symbol.Reference; +import org.teiid.query.sql.symbol.ScalarSubquery; +import org.teiid.query.sql.util.SymbolMap; +import org.teiid.query.sql.visitor.AggregateSymbolCollectorVisitor; +import org.teiid.query.sql.visitor.CommandCollectorVisitor; +import org.teiid.query.sql.visitor.ExpressionMappingVisitor; +import org.teiid.query.sql.visitor.FunctionCollectorVisitor; +import org.teiid.query.sql.visitor.GroupsUsedByElementsVisitor; +import org.teiid.query.sql.visitor.ReferenceCollectorVisitor; +import org.teiid.query.sql.visitor.ValueIteratorProviderCollectorVisitor; +import org.teiid.query.util.CommandContext; + +public final class RulePlanSubqueries implements OptimizerRule { + + private static final int LARGE_INDEPENDENT = 10000; + + /** + * Used to replace correlated references + */ + public static final class ReferenceReplacementVisitor extends + ExpressionMappingVisitor { + private final SymbolMap refs; + private boolean replacedAny; + + public ReferenceReplacementVisitor(SymbolMap refs) { + super(null); + this.refs = refs; + } + + public Expression replaceExpression(Expression element) { + if (element instanceof Reference) { + Reference r = (Reference)element; + Expression ex = refs.getMappedExpression(r.getExpression()); + if (ex != null) { + if (ex instanceof ElementSymbol) { + ElementSymbol es = (ElementSymbol) ex.clone(); + es.setIsExternalReference(false); + ex = es; + } + replacedAny = true; + return ex; + } + } + return element; + } + + } + + public static class PlannedResult { + public List leftExpressions = new LinkedList(); + public List rightExpressions = new LinkedList(); + public Query query; + public boolean not; + public List nonEquiJoinCriteria = new LinkedList(); + public Criteria additionalCritieria; + public Class type; + public boolean mergeJoin; + public boolean madeDistinct; + public boolean makeInd; + public void reset() { + this.leftExpressions.clear(); + this.rightExpressions.clear(); + this.query = null; + this.not = false; + this.nonEquiJoinCriteria.clear(); + this.additionalCritieria = null; + this.type = null; + this.mergeJoin = false; + this.madeDistinct = false; + this.makeInd = false; + } + } + + /** + * Return true if the result from the subquery may be different + * if non-distinct rows are used as input + * @param query + * @return + */ + public static boolean requiresDistinctRows(Query query) { + Set aggs = new HashSet(); + aggs.addAll(AggregateSymbolCollectorVisitor.getAggregates(query.getSelect(), false)); + aggs.addAll(AggregateSymbolCollectorVisitor.getAggregates(query.getHaving(), false)); + if (!aggs.isEmpty() || query.getGroupBy() != null) { + if (!AggregateSymbol.areAggregatesCardinalityDependent(aggs)) { + return false; + } + } else if (query.getSelect().isDistinct()) { + for (Expression projectSymbol : query.getSelect().getProjectedSymbols()) { + Expression ex = SymbolMap.getExpression(projectSymbol); + if (FunctionCollectorVisitor.isNonDeterministic(ex)) { + return true; + } + if (!ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(ex).isEmpty()) { + return true; + } + } + return false; + } + return true; + } + + private IDGenerator idGenerator; + private CapabilitiesFinder capFinder; + private AnalysisRecord analysisRecord; + private CommandContext context; + private QueryMetadataInterface metadata; + private boolean dependent; + + public RulePlanSubqueries(IDGenerator idGenerator, CapabilitiesFinder capFinder, AnalysisRecord analysisRecord, CommandContext context, QueryMetadataInterface metadata) { + this.idGenerator = idGenerator; + this.capFinder = capFinder; + this.analysisRecord = analysisRecord; + this.context = context; + this.metadata = metadata; + } + + /** + * @see OptimizerRule#execute(PlanNode, QueryMetadataInterface, RuleStack) + */ + public PlanNode execute(PlanNode plan, QueryMetadataInterface metadata, CapabilitiesFinder capFinder, RuleStack rules, AnalysisRecord analysisRecord, CommandContext context) + throws QueryPlannerException, TeiidComponentException { + dependent = false; + processSubqueries(plan); + if (dependent) { + rules.push(RuleConstants.PUSH_SELECT_CRITERIA); + } + return plan; + } + + void processSubqueries(PlanNode root) + throws QueryPlannerException, TeiidComponentException { + + PlanNode recurseRoot = root; + if(root.getType() == NodeConstants.Types.SELECT) { + + // Walk to end of the chain and change recurse root + while(recurseRoot.getType() == NodeConstants.Types.SELECT) { + // Look for opportunities to replace with a semi-join + recurseRoot = planMergeJoin(recurseRoot, root); + if (root.getChildCount() == 0) { + root = recurseRoot.getFirstChild(); + if (root.getType() != NodeConstants.Types.SELECT) { + root = root.getParent(); + } + } + recurseRoot = recurseRoot.getFirstChild(); + } + } else if (root.getType() == NodeConstants.Types.PROJECT) { + PlannedResult plannedResult = new PlannedResult(); + List symbols = (List) root.getProperty(Info.PROJECT_COLS); + List output = (List) root.getProperty(Info.OUTPUT_COLS); + for (int i = 0; i < symbols.size(); i++) { + Expression symbol = symbols.get(i); + plannedResult.reset(); + findSubquery(SymbolMap.getExpression(symbol), true, plannedResult); + if (plannedResult.query == null + || plannedResult.query.getFrom() == null + || plannedResult.not) { + continue; + } + PlanNode child = root.getFirstChild(); + PlanNode newRecurseRoot = planMergeJoin(child, child, symbol, plannedResult, true); + if (newRecurseRoot != child) { + Expression newProjection = ((List)newRecurseRoot.getLastChild().getProperty(Info.OUTPUT_COLS)).get(0); + symbols.set(i, newProjection); + output.set(i, newProjection); + ((List)newRecurseRoot.getProperty(Info.OUTPUT_COLS)).add(newProjection); + //we may have used some of the state in planning, so create new + plannedResult = new PlannedResult(); + } + } + } + + if (recurseRoot.getType() != NodeConstants.Types.ACCESS) { + for (PlanNode child : recurseRoot.getChildren()) { + processSubqueries(child); + } + } + } + + /** + * Look for: + * [NOT] EXISTS ( ) + * IN ( ) / SOME ( ) + * + * and replace with a semi join + */ + private PlanNode planMergeJoin(PlanNode current, PlanNode root) throws QueryMetadataException, + TeiidComponentException { + Criteria crit = (Criteria)current.getProperty(NodeConstants.Info.SELECT_CRITERIA); + + PlannedResult plannedResult = findSubquery(crit, true); + if (plannedResult.query == null) { + return current; + } + return planMergeJoin(current, root, crit, plannedResult, false); + } + + private PlanNode planMergeJoin(PlanNode current, PlanNode root, + LanguageObject obj, PlannedResult plannedResult, boolean isProjection) + throws QueryMetadataException, TeiidComponentException { + float sourceCost = NewCalculateCostUtil.computeCostForTree(current.getFirstChild(), metadata); + if (!isProjection && sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE + && sourceCost < RuleChooseDependent.DEFAULT_INDEPENDENT_CARDINALITY && !plannedResult.mergeJoin) { + //TODO: see if a dependent join applies the other direction - which we now handle in the isProjection case + return current; + } + + RelationalPlan originalPlan = (RelationalPlan)plannedResult.query.getProcessorPlan(); + Number originalCardinality = originalPlan.getRootNode().getEstimateNodeCardinality(); + if (!plannedResult.mergeJoin && originalCardinality.floatValue() == NewCalculateCostUtil.UNKNOWN_VALUE) { + //TODO: this check isn't really accurate - exists and scalarsubqueries will always have cardinality 2/1 + //if it's currently unknown, removing criteria won't make it any better + return current; + } + Collection leftGroups = FrameUtil.findJoinSourceNode(current).getGroups(); + if (!planQuery(leftGroups, false, plannedResult)) { + if (plannedResult.mergeJoin && analysisRecord != null && analysisRecord.recordAnnotations()) { + this.analysisRecord.addAnnotation(new Annotation(Annotation.HINTS, "Could not plan as a merge join: " + obj, "ignoring MJ hint", Priority.HIGH)); //$NON-NLS-1$ //$NON-NLS-2$ + } + return current; + } + + //check if the child is already ordered. TODO: see if the ordering is compatible. + PlanNode childSort = NodeEditor.findNodePreOrder(root, NodeConstants.Types.SORT, NodeConstants.Types.SOURCE | NodeConstants.Types.JOIN); + if (childSort != null) { + if (plannedResult.mergeJoin && analysisRecord != null && analysisRecord.recordAnnotations()) { + this.analysisRecord.addAnnotation(new Annotation(Annotation.HINTS, "Could not plan as a merge join since the parent join requires a sort: " + obj, "ignoring MJ hint", Priority.HIGH)); //$NON-NLS-1$ //$NON-NLS-2$ + } + return current; + } + + //add an order by, which hopefully will get pushed down + if (!isProjection) { //we can't sort in the projection case as we can't later decline the sort from the subplan + plannedResult.query.setOrderBy(new OrderBy(plannedResult.rightExpressions).clone()); + for (OrderByItem item : plannedResult.query.getOrderBy().getOrderByItems()) { + int index = plannedResult.query.getProjectedSymbols().indexOf(item.getSymbol()); + if (index >= 0 && !(item.getSymbol() instanceof ElementSymbol)) { + item.setSymbol((Expression) plannedResult.query.getProjectedSymbols().get(index).clone()); + } + item.setExpressionPosition(index); + } + } + + String id = null; + if (isProjection) { + //basic cost determination, without a hint don't proceed if the outer side is "large" + //The other test below will be if a dependent join can be used, which we + //check after planning + if (plannedResult.rightExpressions.isEmpty()) { + return current; + } + if (!plannedResult.mergeJoin) { + if (sourceCost == NewCalculateCostUtil.UNKNOWN_VALUE) { + return current; + } + float ndv = NewCalculateCostUtil.getNDVEstimate(current.getFirstChild(), metadata, sourceCost, plannedResult.leftExpressions, true); + //hard-coded fail-safe + if (ndv > LARGE_INDEPENDENT) { + return current; + } + } + //if the dep join is not created... + id = RuleChooseDependent.nextId(); + PlanNode dep = RuleChooseDependent.getDependentCriteriaNode(id, plannedResult.leftExpressions, plannedResult.rightExpressions, root, metadata, null, false, null); + Criteria crit = (Criteria)dep.getProperty(Info.SELECT_CRITERIA); + plannedResult.query.setCriteria(Criteria.combineCriteria(plannedResult.query.getCriteria(), crit)); + } + + try { + //clone the symbols as they may change during planning + List projectedSymbols = LanguageObject.Util.deepClone(plannedResult.query.getProjectedSymbols(), Expression.class); + //NOTE: we could tap into the relationalplanner at a lower level to get this in a plan node form, + //the major benefit would be to reuse the dependent join planning logic if possible. + RelationalPlan subPlan = (RelationalPlan)QueryOptimizer.optimizePlan(plannedResult.query, metadata, idGenerator, capFinder, analysisRecord, context); + Number planCardinality = subPlan.getRootNode().getEstimateNodeCardinality(); + + if (!plannedResult.mergeJoin) { + if (isProjection) { + RelationalNode rn = subPlan.getRootNode(); + if (!isUsingDependentJoin(id, rn)) { + return current; + } + } else //if we don't have a specific hint, then use costing + if (planCardinality.floatValue() == NewCalculateCostUtil.UNKNOWN_VALUE + || planCardinality.floatValue() > 10000000 + || (sourceCost == NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() > 1000) + || (sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && sourceCost * originalCardinality.floatValue() < planCardinality.floatValue() / (100 * Math.log(Math.max(4, sourceCost))))) { + //bail-out if both are unknown or the new plan is too large + if (analysisRecord != null && analysisRecord.recordDebug()) { + current.recordDebugAnnotation("cost of merge join plan was not favorable", null, "semi merge join will not be used", analysisRecord, metadata); //$NON-NLS-1$ //$NON-NLS-2$ + } + return current; + } + } + + if (isProjection) { + plannedResult.makeInd = false; + } else { + //assume dependent + plannedResult.makeInd = makeDep(sourceCost, planCardinality.floatValue()); + } + + /*if (plannedResult.makeInd + && plannedResult.query.getCorrelatedReferences() == null + && !plannedResult.not + && plannedResult.leftExpressions.size() == 1) { + //TODO: this should just be a dependent criteria node to avoid sorts + }*/ + + current.recordDebugAnnotation("Conditions met (hint or cost)", null, "Converting to a semi merge join", analysisRecord, metadata); //$NON-NLS-1$ //$NON-NLS-2$ + + PlanNode semiJoin = NodeFactory.getNewNode(NodeConstants.Types.JOIN); + semiJoin.addGroups(current.getGroups()); + + //create a proper projection "sub" view. then update the right expressions + //to the virtual group columns + GroupSymbol v = RulePlaceAccess.recontextSymbol(new GroupSymbol("sub"), context.getGroups()); //$NON-NLS-1$ + v.setName(v.getName()); + v.setDefinition(null); + TempMetadataStore tms = new TempMetadataStore(); + Select s = new Select(projectedSymbols); + QueryRewriter.makeSelectUnique(s, false); + v.setMetadataID(tms.addTempGroup(v.getName(), s.getProjectedSymbols())); + List virtualCols = ResolverUtil.resolveElementsInGroup(v, new TempMetadataAdapter(metadata, tms)); + projectedSymbols = virtualCols; + SymbolMap map = SymbolMap.createSymbolMap(virtualCols, s.getProjectedSymbols()); + Map inverseMapping = map.inserseMapping(); + List mappedRight = new ArrayList<>(plannedResult.rightExpressions.size()); + for (Expression ex : (List)plannedResult.rightExpressions) { + ex = inverseMapping.get(SymbolMap.getExpression(ex)); + if (ex == null) { + LogManager.logWarning(LogConstants.CTX_DQP, "Could not map column from subquery optimization, backing out"); //$NON-NLS-1$ + return current; + } + mappedRight.add(ex); + } + plannedResult.rightExpressions = mappedRight; + + Set groups = new LinkedHashSet<>(); + groups.add(v); + semiJoin.addGroups(groups); + semiJoin.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.MERGE); + if (isProjection) { + semiJoin.setProperty(NodeConstants.Info.JOIN_TYPE, JoinType.JOIN_LEFT_OUTER); + //TODO: account for multi-row case + } else { + semiJoin.setProperty(NodeConstants.Info.JOIN_TYPE, plannedResult.not?JoinType.JOIN_ANTI_SEMI:JoinType.JOIN_SEMI); + } + if (!plannedResult.nonEquiJoinCriteria.isEmpty()) { + for (Criteria c : plannedResult.nonEquiJoinCriteria) { + ExpressionMappingVisitor.mapExpressions(c, inverseMapping); + } + semiJoin.setProperty(NodeConstants.Info.NON_EQUI_JOIN_CRITERIA, plannedResult.nonEquiJoinCriteria); + } + List joinCriteria = new ArrayList(); + joinCriteria.addAll(plannedResult.nonEquiJoinCriteria); + for (int i = 0; i < plannedResult.leftExpressions.size(); i++) { + joinCriteria.add(new CompareCriteria((Expression)plannedResult.rightExpressions.get(i), CompareCriteria.EQ, (Expression)plannedResult.leftExpressions.get(i))); + } + semiJoin.setProperty(NodeConstants.Info.JOIN_CRITERIA, joinCriteria); + //nested subqueries are possibly being promoted, so they need their references updated + List refMaps = semiJoin.getAllReferences(); + SymbolMap parentRefs = plannedResult.query.getCorrelatedReferences(); + for (SymbolMap refs : refMaps) { + for (Map.Entry ref : refs.asUpdatableMap().entrySet()) { + Expression expr = ref.getValue(); + if (expr instanceof ElementSymbol) { + Expression convertedExpr = parentRefs.getMappedExpression((ElementSymbol)expr); + if (convertedExpr != null) { + ref.setValue(convertedExpr); + } + } + semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(ref.getValue())); + } + } + semiJoin.setProperty(NodeConstants.Info.LEFT_EXPRESSIONS, plannedResult.leftExpressions); + semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(plannedResult.leftExpressions)); + semiJoin.setProperty(NodeConstants.Info.RIGHT_EXPRESSIONS, plannedResult.rightExpressions); + semiJoin.setProperty(NodeConstants.Info.SORT_RIGHT, SortOption.ALREADY_SORTED); + semiJoin.setProperty(NodeConstants.Info.OUTPUT_COLS, new ArrayList((List)root.getProperty(NodeConstants.Info.OUTPUT_COLS))); + + List childOutput = (List)current.getFirstChild().getProperty(NodeConstants.Info.OUTPUT_COLS); + PlanNode toCorrect = root; + while (toCorrect != current) { + toCorrect.setProperty(NodeConstants.Info.OUTPUT_COLS, childOutput); + toCorrect = toCorrect.getFirstChild(); + } + + PlanNode node = NodeFactory.getNewNode(NodeConstants.Types.ACCESS); + node.setProperty(NodeConstants.Info.PROCESSOR_PLAN, subPlan); + node.setProperty(NodeConstants.Info.OUTPUT_COLS, projectedSymbols); + node.setProperty(NodeConstants.Info.EST_CARDINALITY, planCardinality); + node.addGroups(groups); + root.addAsParent(semiJoin); + semiJoin.addLastChild(node); + PlanNode result = current.getParent(); + if (!isProjection) { + NodeEditor.removeChildNode(result, current); + } + RuleImplementJoinStrategy.insertSort(semiJoin.getFirstChild(), (List) plannedResult.leftExpressions, semiJoin, metadata, capFinder, true, context); + if (isProjection) { //this is always a dep join case, and the predicate was already added + semiJoin.setProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE, id); + semiJoin.setProperty(Info.SORT_RIGHT, SortOption.SORT); + this.dependent = true; + } else if (plannedResult.makeInd && !plannedResult.not) { + id = RuleChooseDependent.nextId(); + //TODO: would like for an enhanced sort merge with the semi dep option to avoid the sorting + //this is a little different than a typical dependent join in that the right is the independent side + PlanNode dep = RuleChooseDependent.getDependentCriteriaNode(id, plannedResult.rightExpressions, plannedResult.leftExpressions, node, metadata, null, false, null); + semiJoin.getFirstChild().addAsParent(dep); + semiJoin.setProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE, id); + this.dependent = true; + } + return result; + } catch (QueryPlannerException e) { + //can't be done - probably access patterns - what about dependent + return current; + } + } + + /** + * Because we lack a cost value based upon complexity the + * heurstic is to look that the dependent set is fully pushed. + * there are situations where it won't be fully pushed, but + * for which it will still be a better plan and we'll reject + */ + private boolean isUsingDependentJoin(String id, RelationalNode rn) { + if (rn instanceof DependentAccessNode) { + DependentAccessNode dan = (DependentAccessNode)rn; + Query qc = (Query) dan.getCommand(); + Criteria c = qc.getCriteria(); + for (Criteria crit : Criteria.separateCriteriaByAnd(c)) { + if (crit instanceof DependentSetCriteria) { + DependentSetCriteria dsc = (DependentSetCriteria)crit; + if (dsc.getContextSymbol().equals(id)) { + return true; + } + } + } + } + RelationalNode[] children = rn.getChildren(); + for (int i=0; i leftGroups, boolean requireDistinct, PlannedResult plannedResult) throws QueryMetadataException, TeiidComponentException { + if ((plannedResult.query.getLimit() != null && !plannedResult.query.getLimit().isImplicit()) || plannedResult.query.getFrom() == null) { + return false; + } + + if ((plannedResult.type == ExistsCriteria.class || plannedResult.type == ScalarSubquery.class) && plannedResult.query.getCorrelatedReferences() == null) { + //we can't really improve on this case + //TODO: do this check earlier + return false; + } + + plannedResult.query = (Query)plannedResult.query.clone(); + for (Command c : CommandCollectorVisitor.getCommands(plannedResult.query)) { + //subqueries either need to be re-resolved or replanned to maintain + //multilevel correlated references. it's easier for now to replan + c.setProcessorPlan(null); + } + plannedResult.query.setLimit(null); + + List rightGroups = plannedResult.query.getFrom().getGroups(); + Set requiredExpressions = new LinkedHashSet(); + final SymbolMap refs = plannedResult.query.getCorrelatedReferences(); + boolean addGroupBy = false; + if (refs != null) { + boolean hasAggregates = plannedResult.query.hasAggregates(); + Criteria where = plannedResult.query.getCriteria(); + if (plannedResult.query.getGroupBy() == null) { + plannedResult.query.setCriteria(null); + } + Criteria having = plannedResult.query.getHaving(); + plannedResult.query.setHaving(null); + if (hasCorrelatedReferences(plannedResult.query, refs)) { + return false; + } + if (plannedResult.query.getGroupBy() == null) { + processCriteria(leftGroups, plannedResult, rightGroups, requiredExpressions, refs, where, null, true); + if (hasAggregates) { + if (!plannedResult.nonEquiJoinCriteria.isEmpty()) { + return false; + } + addGroupBy = true; + if (!canAddGrouping(plannedResult.query.getSelect())) { + return false; + } + if (!canAddGrouping(having)) { + boolean okToAdd = false; + for (Expression ex : (List)plannedResult.rightExpressions) { + if (canAddGrouping(ex)) { + okToAdd = true; + } + } + if (!okToAdd) { + return false; + } + } + } + } + processCriteria(leftGroups, plannedResult, rightGroups, requiredExpressions, refs, having, plannedResult.query.getGroupBy(), false); + } + + if (plannedResult.additionalCritieria != null) { + RuleChooseJoinStrategy.separateCriteria(leftGroups, rightGroups, plannedResult.leftExpressions, plannedResult.rightExpressions, Criteria.separateCriteriaByAnd(plannedResult.additionalCritieria), plannedResult.nonEquiJoinCriteria); + } + + if (plannedResult.leftExpressions.isEmpty()) { + return false; + } + + plannedResult.leftExpressions = RuleChooseJoinStrategy.createExpressionSymbols(plannedResult.leftExpressions); + plannedResult.rightExpressions = RuleChooseJoinStrategy.createExpressionSymbols(plannedResult.rightExpressions); + + if (requireDistinct && !addGroupBy) { + //ensure that uniqueness applies to the in condition + if (plannedResult.rightExpressions.size() > 1 + && (plannedResult.type != SubquerySetCriteria.class || !isDistinct(plannedResult.query, plannedResult.rightExpressions.subList(plannedResult.rightExpressions.size() - 1, plannedResult.rightExpressions.size()), metadata))) { + return false; + } + + if (!isDistinct(plannedResult.query, plannedResult.rightExpressions, metadata)) { + if (plannedResult.type == ExistsCriteria.class) { + if (requiredExpressions.size() > plannedResult.leftExpressions.size()) { + return false; //not an equi join + } + } else if (!requiredExpressions.isEmpty() && !isDistinct(plannedResult.query, plannedResult.query.getProjectedSymbols(), metadata)) { + return false; + } + plannedResult.query.getSelect().setDistinct(true); + plannedResult.madeDistinct = true; + } + } + + //it doesn't matter what the select columns are + if (plannedResult.type == ExistsCriteria.class) { + plannedResult.query.getSelect().clearSymbols(); + } + + if (addGroupBy) { + LinkedHashSet groupingSymbols = new LinkedHashSet(); + for (Expression expr : (List)plannedResult.rightExpressions) { + AggregateSymbolCollectorVisitor.getAggregates(expr, null, groupingSymbols, null, null, null); + } + if (!groupingSymbols.isEmpty()) { + plannedResult.query.setGroupBy((GroupBy) new GroupBy(new ArrayList(groupingSymbols)).clone()); + } + } + HashSet projectedSymbols = new HashSet(); + for (Expression ses : plannedResult.query.getProjectedSymbols()) { + projectedSymbols.add(SymbolMap.getExpression(ses)); + } + for (Expression ses : requiredExpressions) { + if (projectedSymbols.add(ses)) { + plannedResult.query.getSelect().addSymbol((Expression)ses.clone()); + } + } + for (Expression ses : (List)plannedResult.rightExpressions) { + if (projectedSymbols.add(SymbolMap.getExpression(ses))) { + plannedResult.query.getSelect().addSymbol((Expression)ses.clone()); + } + } + return true; + } + + private boolean canAddGrouping(LanguageObject lo) { + for (AggregateSymbol as : AggregateSymbolCollectorVisitor.getAggregates(lo, false)) { + if (as.isCount() || as.getAggregateFunction() == Type.TEXTAGG) { + //these can be non-null for no rows. + //TODO: could include udaf as well + return false; + } + } + return true; + } + + private void processCriteria(Collection leftGroups, + PlannedResult plannedResult, List rightGroups, + Set requiredExpressions, final SymbolMap refs, + Criteria joinCriteria, GroupBy groupBy, boolean where) { + if (joinCriteria == null) { + return; + } + List crits = Criteria.separateCriteriaByAnd((Criteria)joinCriteria.clone()); + + for (Iterator critIter = crits.iterator(); critIter.hasNext();) { + Criteria conjunct = critIter.next(); + List additionalRequired = new LinkedList(); + AggregateSymbolCollectorVisitor.getAggregates(conjunct, additionalRequired, additionalRequired, additionalRequired, null, groupBy!=null?groupBy.getSymbols():null); + ReferenceReplacementVisitor emv = new ReferenceReplacementVisitor(refs); + DeepPostOrderNavigator.doVisit(conjunct, emv); + if (!emv.replacedAny) { + //if not correlated, then leave it on the query + critIter.remove(); + if (where) { + plannedResult.query.setCriteria(Criteria.combineCriteria(plannedResult.query.getCriteria(), conjunct)); + } else { + plannedResult.query.setHaving(Criteria.combineCriteria(plannedResult.query.getHaving(), conjunct)); + } + } else { + requiredExpressions.addAll(additionalRequired); + } + } + RuleChooseJoinStrategy.separateCriteria(leftGroups, rightGroups, plannedResult.leftExpressions, plannedResult.rightExpressions, crits, plannedResult.nonEquiJoinCriteria); + } + + public static boolean isDistinct(Query query, List expressions, QueryMetadataInterface metadata) + throws QueryMetadataException, TeiidComponentException { + boolean distinct = false; + if (query.getGroupBy() != null) { + distinct = true; + for (Expression groupByExpr : query.getGroupBy().getSymbols()) { + if (!expressions.contains(groupByExpr)) { + distinct = false; + break; + } + } + } + if (distinct) { + return true; + } + HashSet keyPreservingGroups = new HashSet(); + ResolverUtil.findKeyPreserved(query, keyPreservingGroups, metadata); + return NewCalculateCostUtil.usesKey(expressions, keyPreservingGroups, metadata, true); + } + + private boolean hasCorrelatedReferences(LanguageObject object, SymbolMap correlatedReferences) { + Collection references = ReferenceCollectorVisitor.getReferences(object); + for (Reference reference : references) { + if (correlatedReferences.asMap().containsKey(reference.getExpression())) { + return true; + } + } + return false; + } + + public String toString() { + return "PlanSubqueries"; //$NON-NLS-1$ + } + +} diff --git a/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java b/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java index b82443240c..23db1e3388 100644 --- a/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java +++ b/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java @@ -61,9 +61,9 @@ import org.teiid.query.metadata.TempMetadataStore; import org.teiid.query.optimizer.relational.AliasGenerator; import org.teiid.query.optimizer.relational.rules.NewCalculateCostUtil; -import org.teiid.query.optimizer.relational.rules.RuleMergeCriteria; -import org.teiid.query.optimizer.relational.rules.RuleMergeCriteria.PlannedResult; import org.teiid.query.optimizer.relational.rules.RulePlaceAccess; +import org.teiid.query.optimizer.relational.rules.RulePlanSubqueries; +import org.teiid.query.optimizer.relational.rules.RulePlanSubqueries.PlannedResult; import org.teiid.query.processor.relational.RelationalNodeUtil; import org.teiid.query.resolver.ProcedureContainerResolver; import org.teiid.query.resolver.QueryResolver; @@ -85,7 +85,6 @@ import org.teiid.query.sql.symbol.AggregateSymbol.Type; import org.teiid.query.sql.symbol.WindowFrame.FrameBound; import org.teiid.query.sql.util.SymbolMap; -import org.teiid.query.sql.visitor.AggregateSymbolCollectorVisitor; import org.teiid.query.sql.visitor.CorrelatedReferenceCollectorVisitor; import org.teiid.query.sql.visitor.ElementCollectorVisitor; import org.teiid.query.sql.visitor.EvaluatableVisitor; @@ -93,7 +92,6 @@ import org.teiid.query.sql.visitor.ExpressionMappingVisitor; import org.teiid.query.sql.visitor.FunctionCollectorVisitor; import org.teiid.query.sql.visitor.GroupCollectorVisitor; -import org.teiid.query.sql.visitor.ValueIteratorProviderCollectorVisitor; import org.teiid.query.util.CommandContext; import org.teiid.query.validator.UpdateValidator.UpdateInfo; import org.teiid.query.validator.UpdateValidator.UpdateMapping; @@ -575,7 +573,7 @@ private Command rewriteQuery(Query query) if (from != null) { List symbols = query.getSelect().getSymbols(); - RuleMergeCriteria rmc = new RuleMergeCriteria(null, null, null, this.context, this.metadata); + RulePlanSubqueries rmc = new RulePlanSubqueries(null, null, null, this.context, this.metadata); TreeSet names = new TreeSet(String.CASE_INSENSITIVE_ORDER); List groups = query.getFrom().getGroups(); for (GroupSymbol gs : groups) { @@ -591,7 +589,7 @@ private Command rewriteQuery(Query query) continue; } determineCorrelatedReferences(groups, plannedResult); - boolean requiresDistinct = requiresDistinctRows(query); + boolean requiresDistinct = RulePlanSubqueries.requiresDistinctRows(query); if (!rmc.planQuery(groups, requiresDistinct, plannedResult)) { continue; } @@ -620,7 +618,7 @@ private void rewriteSubqueriesAsJoins(Query query) if (query.getCriteria() == null) { return; } - RuleMergeCriteria rmc = new RuleMergeCriteria(null, null, null, this.context, this.metadata); + RulePlanSubqueries rmc = new RulePlanSubqueries(null, null, null, this.context, this.metadata); List current = Criteria.separateCriteriaByAnd(query.getCriteria()); query.setCriteria(null); List groups = query.getFrom().getGroups(); @@ -635,7 +633,7 @@ private void rewriteSubqueriesAsJoins(Query query) continue; } determineCorrelatedReferences(groups, plannedResult); - boolean requiresDistinct = requiresDistinctRows(query); + boolean requiresDistinct = RulePlanSubqueries.requiresDistinctRows(query); if (!rmc.planQuery(groups, requiresDistinct, plannedResult)) { continue; } @@ -707,29 +705,6 @@ private void determineCorrelatedReferences(List groups, } } - private boolean requiresDistinctRows(Query query) { - Set aggs = new HashSet(); - aggs.addAll(AggregateSymbolCollectorVisitor.getAggregates(query.getSelect(), false)); - aggs.addAll(AggregateSymbolCollectorVisitor.getAggregates(query.getHaving(), false)); - if (!aggs.isEmpty() || query.getGroupBy() != null) { - if (!AggregateSymbol.areAggregatesCardinalityDependent(aggs)) { - return false; - } - } else if (query.getSelect().isDistinct()) { - for (Expression projectSymbol : query.getSelect().getProjectedSymbols()) { - Expression ex = SymbolMap.getExpression(projectSymbol); - if (FunctionCollectorVisitor.isNonDeterministic(ex)) { - return true; - } - if (!ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(ex).isEmpty()) { - return true; - } - } - return false; - } - return true; - } - private Query rewriteGroupBy(Query query) throws TeiidComponentException, TeiidProcessingException { if (query.getGroupBy() == null) { rewriteAggs = false; diff --git a/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java b/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java index 7898feeddb..ae76e6d3aa 100644 --- a/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java +++ b/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java @@ -20,6 +20,7 @@ import static org.junit.Assert.*; import static org.teiid.query.optimizer.TestOptimizer.*; +import static org.teiid.query.processor.TestProcessor.*; import java.util.Arrays; import java.util.List; @@ -1879,4 +1880,91 @@ public void closeSource() { TestProcessor.helpProcess(pp, cc, dataMgr, new List[] {Arrays.asList(new ArrayImpl(1))}); } + /** + * Uses n-many processing despite the hint because of TEIID-5569 + * @throws Exception + */ + @Test public void testSelectSubqueryJoin() throws Exception { + String sql = "SELECT pm1.g1.e1, pm1.g1.e2, /*+ MJ */ (select count(*) from pm1.g2 where e1 = pm1.g1.e1) FROM pm1.g1"; //$NON-NLS-1$ + + QueryMetadataInterface metadata = RealMetadataFactory.example1(); + RealMetadataFactory.setCardinality("pm1.g2", 1000, metadata); + RealMetadataFactory.setCardinality("pm1.g1", 10, metadata); + BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities(); + CommandContext cc = TestProcessor.createCommandContext(); + ProcessorPlan pp = TestProcessor.helpGetPlan(TestOptimizer.helpGetCommand(sql, metadata), metadata, new DefaultCapabilitiesFinder(bsc), cc); + + List[] expected = new List[] { Arrays.asList("a", 1, 3), Arrays.asList("b", 2, 1), Arrays.asList("c", 3, 0) }; + + HardcodedDataManager manager = new HardcodedDataManager(metadata); + manager.addData("SELECT g_0.e1, g_0.e2 FROM g1 AS g_0", new List[] {Arrays.asList("a", 1), Arrays.asList("b", 2), Arrays.asList("c", 3)}); + manager.addData("SELECT 1 FROM g2 AS g_0 WHERE g_0.e1 = 'a'", new List[] {Arrays.asList(1), Arrays.asList(1), Arrays.asList(1)}); + manager.addData("SELECT 1 FROM g2 AS g_0 WHERE g_0.e1 = 'b'", new List[] {Arrays.asList(1)}); + manager.addData("SELECT 1 FROM g2 AS g_0 WHERE g_0.e1 = 'c'", new List[] {}); + + helpProcess(pp, manager, expected); + } + + @Test public void testSelectSubqueryJoin2() throws Exception { + String sql = "SELECT pm1.g1.e1, pm1.g1.e2, (select max(e2) from pm1.g2 where e1 = pm1.g1.e1) FROM pm1.g1"; //$NON-NLS-1$ + + QueryMetadataInterface metadata = RealMetadataFactory.example1(); + RealMetadataFactory.setCardinality("pm1.g2", 1000, metadata); + RealMetadataFactory.setCardinality("pm1.g1", 10, metadata); + BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities(); + CommandContext cc = TestProcessor.createCommandContext(); + ProcessorPlan pp = TestProcessor.helpGetPlan(TestOptimizer.helpGetCommand(sql, metadata), metadata, new DefaultCapabilitiesFinder(bsc), cc); + + List[] expected = new List[] { Arrays.asList("a", 1, 2), Arrays.asList("b", 2, 1), Arrays.asList("c", 3, null) }; + + HardcodedDataManager manager = new HardcodedDataManager(metadata); + manager.addData("SELECT g_0.e1 AS c_0, g_0.e2 AS c_1 FROM g1 AS g_0 ORDER BY c_0", new List[] {Arrays.asList("a", 1), Arrays.asList("b", 2), Arrays.asList("c", 3)}); + manager.addData("SELECT g_0.e1, g_0.e2 FROM g2 AS g_0 WHERE g_0.e1 IN ('a', 'b', 'c')", new List[] {Arrays.asList("a", 1), Arrays.asList("a", 2), Arrays.asList("b", 1)}); + + helpProcess(pp, manager, expected); + } + + @Test public void testTwoSelectSubqueryJoin() throws Exception { + String sql = "SELECT pm1.g1.e1, pm1.g1.e2, (select max(e1) from pm1.g2 where e2 = pm1.g1.e2), (select min(e2) from pm1.g2 where e1 = pm1.g1.e1) FROM pm1.g1"; //$NON-NLS-1$ + + QueryMetadataInterface metadata = RealMetadataFactory.example1(); + RealMetadataFactory.setCardinality("pm1.g2", 1000, metadata); + RealMetadataFactory.setCardinality("pm1.g1", 10, metadata); + BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities(); + CommandContext cc = TestProcessor.createCommandContext(); + ProcessorPlan pp = TestProcessor.helpGetPlan(TestOptimizer.helpGetCommand(sql, metadata), metadata, new DefaultCapabilitiesFinder(bsc), cc); + + List[] expected = new List[] { Arrays.asList("a", 1, "b", 1), Arrays.asList("b", 2, null, 1), Arrays.asList("c", 3, null, 4) }; + + HardcodedDataManager manager = new HardcodedDataManager(metadata); + manager.addData("SELECT g_0.e1 AS c_0, g_0.e2 AS c_1 FROM g1 AS g_0 ORDER BY c_1", new List[] {Arrays.asList("a", 1), Arrays.asList("b", 2), Arrays.asList("c", 3)}); + manager.addData("SELECT g_0.e2, g_0.e1 FROM g2 AS g_0 WHERE g_0.e2 IN (1, 2, 3)", new List[] {Arrays.asList(1, "a"), Arrays.asList(1, "b"), Arrays.asList(1, "a"), Arrays.asList(1, "a")}); + manager.addData("SELECT g_0.e1, g_0.e2 FROM g2 AS g_0 WHERE g_0.e1 IN ('a', 'b', 'c')", new List[] {Arrays.asList("a", 1), Arrays.asList("b", 1), Arrays.asList("a", 1), Arrays.asList("a", 1), Arrays.asList("c", 4)}); + + helpProcess(pp, manager, expected); + } + + @Test public void testSelectSubqueryJoinNotPushed() throws Exception { + String sql = "SELECT pm1.g1.e1, pm1.g1.e2, (select max(e2) from pm1.g2 where e1 = pm1.g1.e1) FROM pm1.g1"; //$NON-NLS-1$ + + QueryMetadataInterface metadata = RealMetadataFactory.example1(); + RealMetadataFactory.setCardinality("pm1.g2", 1000, metadata); + RealMetadataFactory.setCardinality("pm1.g1", 10, metadata); + BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities(); + bsc.setCapabilitySupport(Capability.CRITERIA_IN, false); + CommandContext cc = TestProcessor.createCommandContext(); + ProcessorPlan pp = TestProcessor.helpGetPlan(TestOptimizer.helpGetCommand(sql, metadata), metadata, new DefaultCapabilitiesFinder(bsc), cc); + + List[] expected = new List[] { Arrays.asList("a", 1, 2), Arrays.asList("b", 2, 1), Arrays.asList("c", 3, null) }; + + HardcodedDataManager manager = new HardcodedDataManager(metadata); + manager.addData("SELECT g_0.e1, g_0.e2 FROM g1 AS g_0", new List[] {Arrays.asList("a", 1), Arrays.asList("b", 2), Arrays.asList("c", 3)}); + manager.addData("SELECT g_0.e2 FROM g2 AS g_0 WHERE g_0.e1 = 'a'", new List[] {Arrays.asList(1), Arrays.asList(2)}); + manager.addData("SELECT g_0.e2 FROM g2 AS g_0 WHERE g_0.e1 = 'b'", new List[] {Arrays.asList(1)}); + manager.addData("SELECT g_0.e2 FROM g2 AS g_0 WHERE g_0.e1 = 'c'", new List[] {}); + + helpProcess(pp, manager, expected); + } + } + diff --git a/teiid-feature-pack/wildfly-integration-feature-pack/teiid-releasenotes.html b/teiid-feature-pack/wildfly-integration-feature-pack/teiid-releasenotes.html index 801e345a23..562292e122 100644 --- a/teiid-feature-pack/wildfly-integration-feature-pack/teiid-releasenotes.html +++ b/teiid-feature-pack/wildfly-integration-feature-pack/teiid-releasenotes.html @@ -26,6 +26,7 @@

Highlights

  • TEIID-5351 Updated to the WildFly 14.0.1 server.
  • TEIID-5524 JMX support for Teiid standalone (Thorntail, Spring Boot, embedded)
    • +
  • TEIID-4498 Additional subquery optimization to prevent per-row evaluation.

Compatibility Issues