Skip to content

Commit

Permalink
TEIID-5532
Browse files Browse the repository at this point in the history 
narrowing the commands used to determine usage
correcting the marking of columns as accessible to be localized
  • Loading branch information
@shawkins
shawkins committed Nov 14, 2018
1 parent b26a228 commit 147dc66
Show file tree
Hide file tree
Showing 6 changed files with 81 additions and 39 deletions.
9 changes: 0 additions & 9 deletions engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,6 @@ public enum Type {
private boolean autoIncrement;
private boolean notNull;
private boolean updatable;
private boolean accessed;

/**
* Constructor for group form of metadata ID.
Expand Down Expand Up @@ -448,12 +447,4 @@ public String getName() {
return this.name;
}

public void setAccessed(boolean accessed) {
this.accessed = accessed;
}

public boolean isAccessed() {
return accessed;
}

}
59 changes: 33 additions & 26 deletions engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,8 +299,10 @@ public RelationalPlan optimize(
private void planWith(PlanNode plan, Command command) throws QueryPlannerException,
QueryMetadataException, TeiidComponentException,
QueryResolverException {
//TODO: merge this logic inline with the main rule execution.
RuleStack stack = new RuleStack();
//TODO: merge this logic inline with the main rule execution.
Set<TempMetadataID> accessed = Collections.newSetFromMap(new IdentityHashMap<TempMetadataID, Boolean>());
this.context.setAccessed(accessed);
RuleStack stack = new RuleStack();
stack.push(new RuleAssignOutputElements(false));
if (hints.hasRowBasedSecurity) {
stack.push(new RuleApplySecurity());
Expand All @@ -310,18 +312,10 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti
planner.processWith = false; //we don't want to trigger the with processing for just projection
planner.initialize(command, idGenerator, metadata, capFinder, analysisRecord, context);
planner.executeRules(stack, plan);
//discover all of the usage - taking the top level
//common tables from the user command, and everything else from the plan
List<Command> commands = new ArrayList<>();
if (command instanceof QueryCommand) {
QueryCommand query = (QueryCommand)command;
List<WithQueryCommand> with = query.getWith();
if (with != null) {
for (WithQueryCommand withQueryCommand : with) {
commands.add(withQueryCommand.getCommand());
}
}
}
//discover all of usage not covered in the main plan above
//-- that's WITH clauses and subqueries
LinkedHashSet<Command> commands = new LinkedHashSet<>();
collectWithCommands(command, commands);
for (PlanNode node : getAllPossibleSubqueryNodes(plan)) {
List<SubqueryContainer<?>> subqueryContainers = node.getSubqueryContainers();
for (SubqueryContainer<?> subqueryContainer : subqueryContainers) {
Expand All @@ -330,13 +324,18 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti
if (node.getType() == NodeConstants.Types.SOURCE) {
Command nested = (Command) node.getProperty(Info.NESTED_COMMAND);
if (nested != null) {
commands.add(nested);
collectWithCommands(nested, commands);
}
}
}
while (!commands.isEmpty()) {
Command cmd = commands.remove(commands.size() - 1);
commands.addAll(CommandCollectorVisitor.getCommands(cmd, true));
List<Command> toExplore = new ArrayList<>(commands);
while (!toExplore.isEmpty()) {
Command cmd = toExplore.remove(toExplore.size() - 1);
for (Command subCommand : CommandCollectorVisitor.getCommands(cmd, true)) {
if (commands.add(subCommand)) {
toExplore.add(subCommand);
}
}
try {
PlanNode temp = planner.generatePlan((Command) cmd.clone());
stack.push(new RuleAssignOutputElements(false));
Expand All @@ -354,12 +353,7 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti
tid.getTableData().setModel(null);
}

//TODO: we should only minimize the projection for with clauses
//that are local to the current command.
//cte's in views are effectively causing us to repeat this
//analysis every time, as the logic doesn't consider
//transitive column usage
subCommand = minimizeWithProjection(with, subCommand, tid);
subCommand = minimizeWithProjection(with, subCommand, tid, accessed);
if (with.isRecursive()) {
SetQuery setQuery = (SetQuery) subCommand;

Expand Down Expand Up @@ -422,21 +416,34 @@ private void planWith(PlanNode plan, Command command) throws QueryPlannerExcepti
((TempMetadataID)with.getGroupSymbol().getMetadataID()).getTableData().setModel(modelID);
this.withPlanningState.pushdownWith.put(with.getGroupSymbol().getName(), wqc);
}
this.context.setAccessed(null);
}

private void collectWithCommands(Command command, Collection<Command> commands) {
if (command instanceof QueryCommand) {
QueryCommand query = (QueryCommand)command;
List<WithQueryCommand> with = query.getWith();
if (with != null) {
for (WithQueryCommand withQueryCommand : with) {
commands.add(withQueryCommand.getCommand());
}
}
}
}

private List<PlanNode> getAllPossibleSubqueryNodes(PlanNode plan) {
return NodeEditor.findAllNodes(plan, NodeConstants.Types.PROJECT | NodeConstants.Types.SELECT | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE | NodeConstants.Types.GROUP | NodeConstants.Types.SORT);
}

private QueryCommand minimizeWithProjection(WithQueryCommand with,
QueryCommand subCommand, TempMetadataID tid)
QueryCommand subCommand, TempMetadataID tid, Collection<TempMetadataID> accessed)
throws QueryMetadataException, QueryResolverException,
TeiidComponentException {
List<TempMetadataID> elements = tid.getElements();
List<Integer> toRemove = new ArrayList<Integer>();
for (int i = elements.size()-1; i >= 0; i--) {
TempMetadataID elem = elements.get(i);
if (!elem.isAccessed()) {
if (!accessed.contains(elem)) {
toRemove.add(i);
}
}
Expand Down
6 changes: 6 additions & 0 deletions engine/src/main/java/org/teiid/query/optimizer/relational/plantree/PlanNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.teiid.query.metadata.QueryMetadataInterface;
import org.teiid.query.optimizer.relational.plantree.NodeConstants.Info;
import org.teiid.query.sql.LanguageObject;
import org.teiid.query.sql.lang.Command;
import org.teiid.query.sql.lang.Criteria;
import org.teiid.query.sql.lang.OrderBy;
import org.teiid.query.sql.lang.SubqueryContainer;
Expand Down Expand Up @@ -405,6 +406,11 @@ public List<SubqueryContainer<?>> getSubqueryContainers() {
TableFunctionReference tfr = (TableFunctionReference)this.getProperty(NodeConstants.Info.TABLE_FUNCTION);
if (tfr != null) {
toSearch = Arrays.asList(tfr);
} else {
Command cmd = (Command) this.getProperty(Info.VIRTUAL_COMMAND);
if (cmd != null) {
toSearch = Arrays.asList(cmd);
}
}
break;
}
Expand Down
2 changes: 1 addition & 1 deletion ...ne/src/main/java/org/teiid/query/optimizer/relational/rules/RuleAssignOutputElements.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ private void assignOutputElements(PlanNode root, List<Expression> outputElements
if (ex instanceof ElementSymbol) {
Object id = ((ElementSymbol)ex).getMetadataID();
if (id instanceof TempMetadataID) {
((TempMetadataID) id).setAccessed(true);
context.addAccessed((TempMetadataID)id);
}
}
}
Expand Down
18 changes: 17 additions & 1 deletion engine/src/main/java/org/teiid/query/util/CommandContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
import org.teiid.query.QueryPlugin;
import org.teiid.query.metadata.QueryMetadataInterface;
import org.teiid.query.metadata.TempMetadataAdapter;
import org.teiid.query.metadata.TempMetadataID;
import org.teiid.query.parser.ParseInfo;
import org.teiid.query.processor.QueryProcessor;
import org.teiid.query.sql.symbol.ElementSymbol;
Expand Down Expand Up @@ -224,6 +225,8 @@ private static class GlobalState implements Cloneable {
private Long currentTimestamp;
private boolean atomicBlock;

private Collection<TempMetadataID> accessed;

/**
* Construct a new context.
*/
Expand Down Expand Up @@ -1228,5 +1231,18 @@ public boolean isAtomicBlock() {
public void setAtomicBlock(boolean atomicBlock) {
this.atomicBlock = atomicBlock;
}


/**
* Used by the planner to track only what is accessed by the plan
*/
public void addAccessed(TempMetadataID id) {
if (this.accessed != null) {
this.accessed.add(id);
}
}

public void setAccessed(Collection<TempMetadataID> accessed) {
this.accessed = accessed;
}

}
26 changes: 24 additions & 2 deletions engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -555,11 +555,11 @@ public TupleSource registerRequest(CommandContext context,

String sql = "SELECT (with b (x) as /*+ no_inline */ (select e1 from pm1.g1) select b.x || c.x from b,b b1), x from (with a (x, b, c) as /*+ no_inline */ (select e1, e2, e3 from pm1.g1) select * from a limit 1) as c"; //$NON-NLS-1$

TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, b, c) AS /*+ no_inline */ (SELECT g_0.e1, g_0.e2, g_0.e3 FROM pm1.g1 AS g_0) SELECT (WITH b (x) AS /*+ no_inline */ (SELECT g_1.e1 FROM pm1.g1 AS g_1) SELECT concat(g_2.x, v_0.c_0) FROM b AS g_2, b AS g_3), v_0.c_0 FROM (SELECT g_0.x AS c_0 FROM a AS g_0 LIMIT 1) AS v_0"}, ComparisonMode.EXACT_COMMAND_STRING);
TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, b, c) AS /*+ no_inline */ (SELECT g_0.e1, null, UNKNOWN FROM pm1.g1 AS g_0) SELECT (WITH b (x) AS /*+ no_inline */ (SELECT g_1.e1 FROM pm1.g1 AS g_1) SELECT concat(g_2.x, v_0.c_0) FROM b AS g_2, b AS g_3), v_0.c_0 FROM (SELECT g_0.x AS c_0 FROM a AS g_0 LIMIT 1) AS v_0"}, ComparisonMode.EXACT_COMMAND_STRING);

caps.setCapabilitySupport(Capability.SUBQUERY_COMMON_TABLE_EXPRESSIONS, false);

TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, b, c) AS /*+ no_inline */ (SELECT g_0.e1, g_0.e2, g_0.e3 FROM pm1.g1 AS g_0) SELECT g_0.x AS c_0 FROM a AS g_0 LIMIT 1"}, ComparisonMode.EXACT_COMMAND_STRING);
TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, b, c) AS /*+ no_inline */ (SELECT g_0.e1, null, UNKNOWN FROM pm1.g1 AS g_0) SELECT g_0.x AS c_0 FROM a AS g_0 LIMIT 1"}, ComparisonMode.EXACT_COMMAND_STRING);

sql = "SELECT (with b (x) as (select e1 from pm1.g1) select b.x || c.x from b,b b1), x from (with a (x, b, c) as (select e1, e2, e3 from pm1.g1) select * from a limit 1) as c"; //$NON-NLS-1$

Expand Down Expand Up @@ -1078,6 +1078,28 @@ public TupleSource registerRequest(CommandContext context,
hdm.addData("SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0", Arrays.asList("a", 1, true));
TestProcessor.helpProcess(plan, hdm, new List<?>[] {Arrays.asList(new ArrayImpl(true))});
}

@Test public void testMultpleViewUsage() throws Exception {
CommandContext cc = TestProcessor.createCommandContext();
BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities();

TransformationMetadata metadata = RealMetadataFactory.fromDDL("create foreign table g1 (e1 string, e2 integer, e3 boolean);"
+ " create view v as with CTE1 as /*+ no_inline */ (SELECT e1, e2, e3 from pm1.g1) select * from cte1", "x", "pm1");

String sql = "select * from v";

ProcessorPlan plan = helpGetPlan(helpParse(sql), metadata, new DefaultCapabilitiesFinder(bsc), cc);
HardcodedDataManager hdm = new HardcodedDataManager(metadata);
hdm.addData("SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0", Arrays.asList("a", 1, true));
TestProcessor.helpProcess(plan, hdm, new List<?>[] {Arrays.asList("a", 1, true)});

//should reset the notion of "accessed" to local command
sql = "select e1 from v";
plan = helpGetPlan(helpParse(sql), metadata, new DefaultCapabilitiesFinder(bsc), cc);
hdm = new HardcodedDataManager(metadata);
hdm.addData("SELECT g_0.e1 FROM g1 AS g_0", Arrays.asList("a"));
TestProcessor.helpProcess(plan, hdm, new List<?>[] {Arrays.asList("a")});
}

@Test public void testNestedSubqueryPreeval() throws Exception {
CommandContext cc = TestProcessor.createCommandContext();
Expand Down

0 comments on commit 147dc66

Please sign in to comment.