Skip to content

Commit

Permalink
TEIID-5532
Browse files Browse the repository at this point in the history 
further fix for inappropriate projection minimization when the cte is
nested in a view
  • Loading branch information
@shawkins
shawkins committed Nov 8, 2018
1 parent 6659110 commit 3c1774e
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 28 deletions.
88 changes: 64 additions & 24 deletions engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
View file
Expand Up @@ -310,8 +310,30 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti
planner.processWith = false; //we don't want to trigger the with processing for just projection
planner.initialize(command, idGenerator, metadata, capFinder, analysisRecord, context);
planner.executeRules(stack, plan);
//discover all of the usage
List<Command> commands = CommandCollectorVisitor.getCommands(command, true);
//discover all of the usage - taking the top level
//common tables from the user command, and everything else from the plan
List<Command> commands = new ArrayList<>();
if (command instanceof QueryCommand) {
QueryCommand query = (QueryCommand)command;
List<WithQueryCommand> with = query.getWith();
if (with != null) {
for (WithQueryCommand withQueryCommand : with) {
commands.add(withQueryCommand.getCommand());
}
}
}
for (PlanNode node : getAllPossibleSubqueryNodes(plan)) {
List<SubqueryContainer<?>> subqueryContainers = node.getSubqueryContainers();
for (SubqueryContainer<?> subqueryContainer : subqueryContainers) {
commands.add(subqueryContainer.getCommand());
}
if (node.getType() == NodeConstants.Types.SOURCE) {
Command nested = (Command) node.getProperty(Info.NESTED_COMMAND);
if (nested != null) {
commands.add(nested);
}
}
}
while (!commands.isEmpty()) {
Command cmd = commands.remove(commands.size() - 1);
commands.addAll(CommandCollectorVisitor.getCommands(cmd, true));
Expand All @@ -331,27 +353,13 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti
if (tid.getTableData().getModel() != TempMetadataAdapter.TEMP_MODEL) {
tid.getTableData().setModel(null);
}
List<TempMetadataID> elements = tid.getElements();
List<Integer> toRemove = new ArrayList<Integer>();
for (int i = elements.size()-1; i >= 0; i--) {
TempMetadataID elem = elements.get(i);
if (!elem.isAccessed()) {
toRemove.add(i);
}
}
//the strategy here is to replace the actual projections with null. this keeps
//the definition of the with clause consistent
if (!toRemove.isEmpty()) {
if (with.isRecursive()) {
SetQuery setQuery = (SetQuery) subCommand;
setQuery.setLeftQuery(removeUnusedProjection(with, setQuery.getLeftQuery(), elements, toRemove));
setQuery.setRightQuery(removeUnusedProjection(with, setQuery.getRightQuery(), elements, toRemove));
} else {
subCommand = removeUnusedProjection(with, subCommand, elements,
toRemove);
with.setCommand(subCommand);
}
}

//TODO: we should only minimize the projection for with clauses
//that are local to the current command.
//cte's in views are effectively causing us to repeat this
//analysis every time, as the logic doesn't consider
//transitive column usage
subCommand = minimizeWithProjection(with, subCommand, tid);
if (with.isRecursive()) {
SetQuery setQuery = (SetQuery) subCommand;

Expand Down Expand Up @@ -416,6 +424,38 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti
}
}

private List<PlanNode> getAllPossibleSubqueryNodes(PlanNode plan) {
return NodeEditor.findAllNodes(plan, NodeConstants.Types.PROJECT | NodeConstants.Types.SELECT | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE | NodeConstants.Types.GROUP | NodeConstants.Types.SORT);
}

private QueryCommand minimizeWithProjection(WithQueryCommand with,
QueryCommand subCommand, TempMetadataID tid)
throws QueryMetadataException, QueryResolverException,
TeiidComponentException {
List<TempMetadataID> elements = tid.getElements();
List<Integer> toRemove = new ArrayList<Integer>();
for (int i = elements.size()-1; i >= 0; i--) {
TempMetadataID elem = elements.get(i);
if (!elem.isAccessed()) {
toRemove.add(i);
}
}
//the strategy here is to replace the actual projections with null. this keeps
//the definition of the with clause consistent
if (!toRemove.isEmpty()) {
if (with.isRecursive()) {
SetQuery setQuery = (SetQuery) subCommand;
setQuery.setLeftQuery(removeUnusedProjection(with, setQuery.getLeftQuery(), elements, toRemove));
setQuery.setRightQuery(removeUnusedProjection(with, setQuery.getRightQuery(), elements, toRemove));
} else {
subCommand = removeUnusedProjection(with, subCommand, elements,
toRemove);
with.setCommand(subCommand);
}
}
return subCommand;
}

/**
* Remove unused projects by replacing with null
* @param with
Expand Down Expand Up @@ -724,7 +764,7 @@ public void initialize(Command command, IDGenerator idGenerator,
}

private void connectSubqueryContainers(PlanNode plan, boolean skipPlanning) throws QueryPlannerException, QueryMetadataException, TeiidComponentException {
for (PlanNode node : NodeEditor.findAllNodes(plan, NodeConstants.Types.PROJECT | NodeConstants.Types.SELECT | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE | NodeConstants.Types.GROUP | NodeConstants.Types.SORT)) {
for (PlanNode node : getAllPossibleSubqueryNodes(plan)) {
Set<GroupSymbol> groupSymbols = getGroupSymbols(node);
List<SubqueryContainer<?>> subqueryContainers = node.getSubqueryContainers();
planSubqueries(groupSymbols, node, subqueryContainers, false, skipPlanning);
Expand Down
93 changes: 89 additions & 4 deletions engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
View file
Expand Up @@ -204,9 +204,6 @@ public class TestWithClauseProcessing {
String sql = "with a as /*+ no_inline */ (select x, y, z from (select e1 as x, e2 as y, e3 as z from pm1.g1) v), b as /*+ no_inline */ (select e4 from pm1.g3) SELECT count(a.x), max(a.y) from a, a z group by z.x having max(a.y) < (with b as /*+ no_inline */ (select e1 from pm1.g1) select a.y from a, b where a.x = z.x)"; //$NON-NLS-1$

HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached());
List<?>[] expected = new List[] {
Arrays.asList("a", 1, "a"),
};

dataManager.addData("SELECT g_0.e1, g_0.e2 FROM g1 AS g_0", Arrays.asList("a", 1));
dataManager.addData("WITH b__3 (e1) AS (SELECT NULL FROM g1 AS g_0) SELECT 1 FROM b__3 AS g_0", Arrays.asList(1));
Expand Down Expand Up @@ -1063,7 +1060,24 @@ public TupleSource registerRequest(CommandContext context,
hdm.addData("SELECT g_0.e1, g_0.e2 FROM g2 AS g_0", Arrays.asList("a", 1));
hdm.addData("SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0", Arrays.asList("a", 1, true));
TestProcessor.helpProcess(plan, hdm, new List<?>[] {Arrays.asList(new ArrayImpl(true))});
}
}

@Test public void testUseInProjectedSubqueryView() throws Exception {
CommandContext cc = TestProcessor.createCommandContext();
BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities();

TransformationMetadata metadata = RealMetadataFactory.fromDDL("create foreign table g1 (e1 string, e2 integer, e3 boolean);"
+ " create foreign table g2 (e1 string, e2 integer, e3 boolean);"
+ " create view v as with CTE1 as /*+ no_inline */ (SELECT e1, e2, e3 from pm1.g1) select array_agg((select e3 from cte1 where e1=pm1.g2.e1 and e2=pm1.g2.e2)) from pm1.g2", "x", "pm1");

String sql = "select * from v";

ProcessorPlan plan = helpGetPlan(helpParse(sql), metadata, new DefaultCapabilitiesFinder(bsc), cc);
HardcodedDataManager hdm = new HardcodedDataManager(metadata);
hdm.addData("SELECT g_0.e1, g_0.e2 FROM g2 AS g_0", Arrays.asList("a", 1));
hdm.addData("SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0", Arrays.asList("a", 1, true));
TestProcessor.helpProcess(plan, hdm, new List<?>[] {Arrays.asList(new ArrayImpl(true))});
}

@Test public void testNestedSubqueryPreeval() throws Exception {
CommandContext cc = TestProcessor.createCommandContext();
Expand Down Expand Up @@ -1344,6 +1358,77 @@ public TupleSource registerRequest(CommandContext context,
dataManager.addData("EXEC logMsg('INFO', 'DEBUG.FOO.BAR', 'baltimore')", Arrays.asList(Boolean.TRUE));
helpProcess(plan, cc, dataManager, new List[] {Arrays.asList(Boolean.TRUE)});
}

@Test public void testWithProjectionMinimizationInView() throws Exception {
CommandContext cc = TestProcessor.createCommandContext();
BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities();

String ddl = "CREATE foreign TABLE tab1\n" +
"(\n" +
" name string(4000)\n" +
");\n" +
"CREATE foreign TABLE tab2\n" +
"(\n" +
" course_id integer\n" +
");\n" +
"CREATE foreign TABLE tab3\n" +
"(\n" +
" course_id long\n" +
");\n"
+ "create view instructor_statement_2_3 as\n" +
"with base_data as (\n" +
" select\n" +
" instructor_payment.course_id\n" +
" from dwh.tab3 as instructor_payment\n" +
")\n" +
" ,adjustments as (\n" +
" select\n" +
" course_id\n" +
" from dwh.tab2\n" +
")\n" +
" ,union_data as (\n" +
" select\n" +
" course_id\n" +
" from base_data \n" +
" union\n" +
" select\n" +
" course_id\n" +
" from base_data \n" +
" union\n" +
" select\n" +
" course_id\n" +
" from adjustments \n" +
")\n" +
" ,line_items as (\n" +
" select\n" +
" dim_playlist.name as playlist_name\n" +
" from union_data\n" +
" left join dwh.tab1 as dim_playlist on true\n" +
")\n" +
" ,sub_totals as (\n" +
" select\n" +
" playlist_name\n" +
" from line_items\n" +
")\n" +
"select\n" +
" line_items.playlist_name \n" +
"from line_items\n" +
"union\n" +
"select\n" +
" sub_totals.playlist_name\n" +
"from sub_totals";

TransformationMetadata metadata = RealMetadataFactory.fromDDL(ddl, "x", "dwh");

String sql = "select * from instructor_statement_2_3";

ProcessorPlan plan = helpGetPlan(helpParse(sql), metadata, new DefaultCapabilitiesFinder(bsc), cc);
HardcodedDataManager hdm = new HardcodedDataManager(metadata);
hdm.addData("SELECT g_0.course_id FROM tab2 AS g_0", Arrays.asList(1));
hdm.addData("SELECT g_0.name FROM tab1 AS g_0", Arrays.asList("a"));
hdm.addData("SELECT g_0.course_id FROM tab3 AS g_0", Arrays.asList(1l));
TestProcessor.helpProcess(plan, hdm, new List<?>[] {Arrays.asList("a")});
}

}

0 comments on commit 3c1774e

Please sign in to comment.