diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java b/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java index 91ea02a122..b851368adc 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java @@ -310,8 +310,30 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti planner.processWith = false; //we don't want to trigger the with processing for just projection planner.initialize(command, idGenerator, metadata, capFinder, analysisRecord, context); planner.executeRules(stack, plan); - //discover all of the usage - List commands = CommandCollectorVisitor.getCommands(command, true); + //discover all of the usage - taking the top level + //common tables from the user command, and everything else from the plan + List commands = new ArrayList<>(); + if (command instanceof QueryCommand) { + QueryCommand query = (QueryCommand)command; + List with = query.getWith(); + if (with != null) { + for (WithQueryCommand withQueryCommand : with) { + commands.add(withQueryCommand.getCommand()); + } + } + } + for (PlanNode node : getAllPossibleSubqueryNodes(plan)) { + List> subqueryContainers = node.getSubqueryContainers(); + for (SubqueryContainer subqueryContainer : subqueryContainers) { + commands.add(subqueryContainer.getCommand()); + } + if (node.getType() == NodeConstants.Types.SOURCE) { + Command nested = (Command) node.getProperty(Info.NESTED_COMMAND); + if (nested != null) { + commands.add(nested); + } + } + } while (!commands.isEmpty()) { Command cmd = commands.remove(commands.size() - 1); commands.addAll(CommandCollectorVisitor.getCommands(cmd, true)); @@ -331,27 +353,13 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti if (tid.getTableData().getModel() != TempMetadataAdapter.TEMP_MODEL) { tid.getTableData().setModel(null); } - List elements = tid.getElements(); - List toRemove = new ArrayList(); - for (int i = elements.size()-1; i >= 0; i--) { - TempMetadataID elem = elements.get(i); - if (!elem.isAccessed()) { - toRemove.add(i); - } - } - //the strategy here is to replace the actual projections with null. this keeps - //the definition of the with clause consistent - if (!toRemove.isEmpty()) { - if (with.isRecursive()) { - SetQuery setQuery = (SetQuery) subCommand; - setQuery.setLeftQuery(removeUnusedProjection(with, setQuery.getLeftQuery(), elements, toRemove)); - setQuery.setRightQuery(removeUnusedProjection(with, setQuery.getRightQuery(), elements, toRemove)); - } else { - subCommand = removeUnusedProjection(with, subCommand, elements, - toRemove); - with.setCommand(subCommand); - } - } + + //TODO: we should only minimize the projection for with clauses + //that are local to the current command. + //cte's in views are effectively causing us to repeat this + //analysis every time, as the logic doesn't consider + //transitive column usage + subCommand = minimizeWithProjection(with, subCommand, tid); if (with.isRecursive()) { SetQuery setQuery = (SetQuery) subCommand; @@ -416,6 +424,38 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti } } + private List getAllPossibleSubqueryNodes(PlanNode plan) { + return NodeEditor.findAllNodes(plan, NodeConstants.Types.PROJECT | NodeConstants.Types.SELECT | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE | NodeConstants.Types.GROUP | NodeConstants.Types.SORT); + } + + private QueryCommand minimizeWithProjection(WithQueryCommand with, + QueryCommand subCommand, TempMetadataID tid) + throws QueryMetadataException, QueryResolverException, + TeiidComponentException { + List elements = tid.getElements(); + List toRemove = new ArrayList(); + for (int i = elements.size()-1; i >= 0; i--) { + TempMetadataID elem = elements.get(i); + if (!elem.isAccessed()) { + toRemove.add(i); + } + } + //the strategy here is to replace the actual projections with null. this keeps + //the definition of the with clause consistent + if (!toRemove.isEmpty()) { + if (with.isRecursive()) { + SetQuery setQuery = (SetQuery) subCommand; + setQuery.setLeftQuery(removeUnusedProjection(with, setQuery.getLeftQuery(), elements, toRemove)); + setQuery.setRightQuery(removeUnusedProjection(with, setQuery.getRightQuery(), elements, toRemove)); + } else { + subCommand = removeUnusedProjection(with, subCommand, elements, + toRemove); + with.setCommand(subCommand); + } + } + return subCommand; + } + /** * Remove unused projects by replacing with null * @param with @@ -724,7 +764,7 @@ public void initialize(Command command, IDGenerator idGenerator, } private void connectSubqueryContainers(PlanNode plan, boolean skipPlanning) throws QueryPlannerException, QueryMetadataException, TeiidComponentException { - for (PlanNode node : NodeEditor.findAllNodes(plan, NodeConstants.Types.PROJECT | NodeConstants.Types.SELECT | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE | NodeConstants.Types.GROUP | NodeConstants.Types.SORT)) { + for (PlanNode node : getAllPossibleSubqueryNodes(plan)) { Set groupSymbols = getGroupSymbols(node); List> subqueryContainers = node.getSubqueryContainers(); planSubqueries(groupSymbols, node, subqueryContainers, false, skipPlanning); diff --git a/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java b/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java index 6715076a6e..6aa0854848 100644 --- a/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java +++ b/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java @@ -204,9 +204,6 @@ public class TestWithClauseProcessing { String sql = "with a as /*+ no_inline */ (select x, y, z from (select e1 as x, e2 as y, e3 as z from pm1.g1) v), b as /*+ no_inline */ (select e4 from pm1.g3) SELECT count(a.x), max(a.y) from a, a z group by z.x having max(a.y) < (with b as /*+ no_inline */ (select e1 from pm1.g1) select a.y from a, b where a.x = z.x)"; //$NON-NLS-1$ HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached()); - List[] expected = new List[] { - Arrays.asList("a", 1, "a"), - }; dataManager.addData("SELECT g_0.e1, g_0.e2 FROM g1 AS g_0", Arrays.asList("a", 1)); dataManager.addData("WITH b__3 (e1) AS (SELECT NULL FROM g1 AS g_0) SELECT 1 FROM b__3 AS g_0", Arrays.asList(1)); @@ -1063,7 +1060,24 @@ public TupleSource registerRequest(CommandContext context, hdm.addData("SELECT g_0.e1, g_0.e2 FROM g2 AS g_0", Arrays.asList("a", 1)); hdm.addData("SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0", Arrays.asList("a", 1, true)); TestProcessor.helpProcess(plan, hdm, new List[] {Arrays.asList(new ArrayImpl(true))}); - } + } + + @Test public void testUseInProjectedSubqueryView() throws Exception { + CommandContext cc = TestProcessor.createCommandContext(); + BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities(); + + TransformationMetadata metadata = RealMetadataFactory.fromDDL("create foreign table g1 (e1 string, e2 integer, e3 boolean);" + + " create foreign table g2 (e1 string, e2 integer, e3 boolean);" + + " create view v as with CTE1 as /*+ no_inline */ (SELECT e1, e2, e3 from pm1.g1) select array_agg((select e3 from cte1 where e1=pm1.g2.e1 and e2=pm1.g2.e2)) from pm1.g2", "x", "pm1"); + + String sql = "select * from v"; + + ProcessorPlan plan = helpGetPlan(helpParse(sql), metadata, new DefaultCapabilitiesFinder(bsc), cc); + HardcodedDataManager hdm = new HardcodedDataManager(metadata); + hdm.addData("SELECT g_0.e1, g_0.e2 FROM g2 AS g_0", Arrays.asList("a", 1)); + hdm.addData("SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0", Arrays.asList("a", 1, true)); + TestProcessor.helpProcess(plan, hdm, new List[] {Arrays.asList(new ArrayImpl(true))}); + } @Test public void testNestedSubqueryPreeval() throws Exception { CommandContext cc = TestProcessor.createCommandContext(); @@ -1344,6 +1358,77 @@ public TupleSource registerRequest(CommandContext context, dataManager.addData("EXEC logMsg('INFO', 'DEBUG.FOO.BAR', 'baltimore')", Arrays.asList(Boolean.TRUE)); helpProcess(plan, cc, dataManager, new List[] {Arrays.asList(Boolean.TRUE)}); } + + @Test public void testWithProjectionMinimizationInView() throws Exception { + CommandContext cc = TestProcessor.createCommandContext(); + BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities(); + + String ddl = "CREATE foreign TABLE tab1\n" + + "(\n" + + " name string(4000)\n" + + ");\n" + + "CREATE foreign TABLE tab2\n" + + "(\n" + + " course_id integer\n" + + ");\n" + + "CREATE foreign TABLE tab3\n" + + "(\n" + + " course_id long\n" + + ");\n" + + "create view instructor_statement_2_3 as\n" + + "with base_data as (\n" + + " select\n" + + " instructor_payment.course_id\n" + + " from dwh.tab3 as instructor_payment\n" + + ")\n" + + " ,adjustments as (\n" + + " select\n" + + " course_id\n" + + " from dwh.tab2\n" + + ")\n" + + " ,union_data as (\n" + + " select\n" + + " course_id\n" + + " from base_data \n" + + " union\n" + + " select\n" + + " course_id\n" + + " from base_data \n" + + " union\n" + + " select\n" + + " course_id\n" + + " from adjustments \n" + + ")\n" + + " ,line_items as (\n" + + " select\n" + + " dim_playlist.name as playlist_name\n" + + " from union_data\n" + + " left join dwh.tab1 as dim_playlist on true\n" + + ")\n" + + " ,sub_totals as (\n" + + " select\n" + + " playlist_name\n" + + " from line_items\n" + + ")\n" + + "select\n" + + " line_items.playlist_name \n" + + "from line_items\n" + + "union\n" + + "select\n" + + " sub_totals.playlist_name\n" + + "from sub_totals"; + + TransformationMetadata metadata = RealMetadataFactory.fromDDL(ddl, "x", "dwh"); + + String sql = "select * from instructor_statement_2_3"; + + ProcessorPlan plan = helpGetPlan(helpParse(sql), metadata, new DefaultCapabilitiesFinder(bsc), cc); + HardcodedDataManager hdm = new HardcodedDataManager(metadata); + hdm.addData("SELECT g_0.course_id FROM tab2 AS g_0", Arrays.asList(1)); + hdm.addData("SELECT g_0.name FROM tab1 AS g_0", Arrays.asList("a")); + hdm.addData("SELECT g_0.course_id FROM tab3 AS g_0", Arrays.asList(1l)); + TestProcessor.helpProcess(plan, hdm, new List[] {Arrays.asList("a")}); + } }