Skip to content

Commit

Permalink
TEIID-5474 determining correlated references prior to with planning
Browse files Browse the repository at this point in the history
  • Loading branch information
@shawkins
shawkins committed Sep 14, 2018
1 parent b772386 commit 8e9c726
Show file tree
Hide file tree
Showing 5 changed files with 167 additions and 23 deletions.
53 changes: 33 additions & 20 deletions engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,15 @@ public RelationalPlan optimize(
throw new QueryPlannerException(e);
}

planWith(plan, command);
if (!this.withPlanningState.withList.isEmpty()) {
// set correlated references before planning with, but
// don't fully plan the subqueries as that needs to happen after
// with planning
connectSubqueryContainers(plan, true);
planWith(plan, command);
}
// Connect ProcessorPlan to SubqueryContainer (if any) of SELECT, PROJECT, and other nodes
connectSubqueryContainers(plan, false); //TODO: merge with node creation

if (plan.getType() == NodeConstants.Types.SOURCE) {
//this was effectively a rewrite
Expand All @@ -215,9 +223,6 @@ public RelationalPlan optimize(
if(debug) {
analysisRecord.println("\nCANONICAL PLAN: \n" + plan); //$NON-NLS-1$
}

// Connect ProcessorPlan to SubqueryContainer (if any) of SELECT or PROJECT nodes
connectSubqueryContainers(plan); //TODO: merge with node creation

// Set top column information on top node
List<Expression> topCols = Util.deepClone(command.getProjectedSymbols(), Expression.class);
Expand Down Expand Up @@ -292,9 +297,6 @@ public RelationalPlan optimize(
private void planWith(PlanNode plan, Command command) throws QueryPlannerException,
QueryMetadataException, TeiidComponentException,
QueryResolverException {
if (this.withPlanningState.withList.isEmpty()) {
return;
}
//TODO: merge this logic inline with the main rule execution.
RuleStack stack = new RuleStack();
stack.push(new RuleAssignOutputElements(false));
Expand Down Expand Up @@ -719,18 +721,18 @@ public void initialize(Command command, IDGenerator idGenerator,
this.context = context;
}

private void connectSubqueryContainers(PlanNode plan) throws QueryPlannerException, QueryMetadataException, TeiidComponentException {
private void connectSubqueryContainers(PlanNode plan, boolean skipPlanning) throws QueryPlannerException, QueryMetadataException, TeiidComponentException {
for (PlanNode node : NodeEditor.findAllNodes(plan, NodeConstants.Types.PROJECT | NodeConstants.Types.SELECT | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE | NodeConstants.Types.GROUP | NodeConstants.Types.SORT)) {
Set<GroupSymbol> groupSymbols = getGroupSymbols(node);
List<SubqueryContainer<?>> subqueryContainers = node.getSubqueryContainers();
planSubqueries(groupSymbols, node, subqueryContainers, false);
planSubqueries(groupSymbols, node, subqueryContainers, false, skipPlanning);
node.addGroups(GroupsUsedByElementsVisitor.getGroups(node.getCorrelatedReferenceElements()));
}
}

public void planSubqueries(
Set<GroupSymbol> groupSymbols,
PlanNode node, List<SubqueryContainer<?>> subqueryContainers, boolean isStackEntry)
PlanNode node, List<SubqueryContainer<?>> subqueryContainers, boolean isStackEntry, boolean skipPlanning)
throws QueryMetadataException, TeiidComponentException,
QueryPlannerException {
if (subqueryContainers.isEmpty()){
Expand All @@ -749,12 +751,11 @@ public void planSubqueries(
continue;
}
//a clone is needed here because the command could get modified during planning
Command subCommand = (Command)container.getCommand().clone();
List<SubqueryContainer<?>> containers = ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(container.getCommand());
List<SubqueryContainer<?>> cloneContainers = null;
if (!containers.isEmpty()) {
cloneContainers = ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(subCommand);
Command subCommand = container.getCommand();
if (!skipPlanning) {
subCommand = (Command)container.getCommand().clone();
}

Set<PlanningStackEntry> entries = null;
PlanningStackEntry stackEntry = null;
if (isStackEntry) {
Expand All @@ -764,19 +765,31 @@ public void planSubqueries(
try {
ArrayList<Reference> correlatedReferences = new ArrayList<Reference>();
CorrelatedReferenceCollectorVisitor.collectReferences(subCommand, localGroupSymbols, correlatedReferences, metadata);
ProcessorPlan procPlan = QueryOptimizer.optimizePlan(subCommand, metadata, idGenerator, capFinder, analysisRecord, context);
container.getCommand().setProcessorPlan(procPlan);
setCorrelatedReferences(container, correlatedReferences);
setCorrelatedReferences(container, correlatedReferences);

List<SubqueryContainer<?>> containers = ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(container.getCommand());
List<SubqueryContainer<?>> cloneContainers = null;

if (!skipPlanning) {
if (!containers.isEmpty()) {
cloneContainers = ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(subCommand);
}
ProcessorPlan procPlan = QueryOptimizer.optimizePlan(subCommand, metadata, idGenerator, capFinder, analysisRecord, context);
container.getCommand().setProcessorPlan(procPlan);
}
//ensure plans are set on the original nested subqueries
if (!containers.isEmpty()) {
for (int i = 0; i < containers.size(); i++) {
Command c = containers.get(i).getCommand();
Command clone = cloneContainers.get(i).getCommand();
List<Reference> refs = new ArrayList<Reference>();
//re-detect the correlated references
CorrelatedReferenceCollectorVisitor.collectReferences(c, localGroupSymbols, refs, metadata);
setCorrelatedReferences(containers.get(i), refs);
c.setProcessorPlan(clone.getProcessorPlan());

if (cloneContainers != null) {
Command clone = cloneContainers.get(i).getCommand();
c.setProcessorPlan(clone.getProcessorPlan());
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion engine/src/main/java/org/teiid/query/optimizer/relational/RowBasedSecurityHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ private static void validateAndPlanSubqueries(LanguageObject object, GroupSymbol
return;
}
Set<GroupSymbol> groups = Collections.singleton(gs);
planner.planSubqueries(groups, null, subqueries, true);
planner.planSubqueries(groups, null, subqueries, true, false);
List<Reference> refs = new LinkedList<Reference>();
CorrelatedReferenceCollectorVisitor.collectReferences(object, groups, refs, planner.metadata);
if (!refs.isEmpty()) {
Expand Down
4 changes: 2 additions & 2 deletions engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleApplySecurity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ public PlanNode execute(PlanNode plan, QueryMetadataInterface metadata,
if (!group.isProcedure()) {
project = RelationalPlanner.createProjectNode(masked);
}
rules.getPlanner().planSubqueries(sourceNode.getGroups(), project, project.getSubqueryContainers(), true);
rules.getPlanner().planSubqueries(sourceNode.getGroups(), project, project.getSubqueryContainers(), true, false);
project.addGroups(GroupsUsedByElementsVisitor.getGroups(project.getCorrelatedReferenceElements()));
if (!group.isProcedure()) {
//we need to insert a view to give a single place to evaluate the subqueries
Expand Down Expand Up @@ -156,7 +156,7 @@ public PlanNode execute(PlanNode plan, QueryMetadataInterface metadata,
if (parent == null) {
parent = critNode;
}
rules.getPlanner().planSubqueries(sourceNode.getGroups(), critNode, critNode.getSubqueryContainers(), true);
rules.getPlanner().planSubqueries(sourceNode.getGroups(), critNode, critNode.getSubqueryContainers(), true, false);
critNode.addGroups(GroupsUsedByElementsVisitor.getGroups(critNode.getCorrelatedReferenceElements()));
root.addAsParent(critNode);
}
Expand Down
58 changes: 58 additions & 0 deletions engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1269,6 +1269,64 @@ public TupleSource registerRequest(CommandContext context,
dataManager.addData("SELECT g_0.column3 AS c_0, g_0.column4 AS c_1 FROM y.table2 AS g_0 WHERE g_0.column3 = 'joe' ORDER BY c_0", Arrays.asList("joe", "employee"));
helpProcess(plan, cc, dataManager, new List[] {Arrays.asList("joe", "employee")});
}

@Test public void testNestedWithWithTopLevelCorrelatedSubquery() throws Exception {
String sql = "SELECT \n" +
"(exec \"logMsg\"(\n" +
" \"level\" => 'INFO',\n" +
" \"context\" => 'DEBUG.FOO.BAR',\n" +
" \"msg\" => tccd.bank_account_holder_name)) as something\n" +
"FROM \n" +
"(\n" +
" SELECT c.city AS \"bank_account_holder_name\"\n" +
" FROM \"address\" c\n" +
" JOIN \n" +
" (\n" +
" select ca.addressid\n" +
" from \"customeraddress\" ca\n" +
" cross JOIN \n" +
" (\n" +
" WITH latest_exchange_rates AS \n" +
" (\n" +
" SELECT exchange_rate_date AS month_begin\n" +
" FROM (\n" +
" SELECT CURDATE() as exchange_rate_date\n" +
" )t\n" +
" )\n" +
" SELECT DISTINCT dt.month_start AS month_begin\n" +
" FROM (\n" +
" CALL createDateDimensionsTable(\n" +
" \"startdate\" => TIMESTAMPADD(SQL_TSI_MONTH, 1, (SELECT month_begin FROM latest_exchange_rates)),\n" +
" \"enddate\" => TIMESTAMPADD(SQL_TSI_YEAR, 15, (SELECT month_begin FROM latest_exchange_rates))\n" +
" ) ) AS dt\n" +
" ) fx\n" +
" ) ci ON ci.addressid = c.addressid\n" +
") tccd ;";

String ddl = "CREATE PROCEDURE createDateDimensionsTable(\n" +
" IN startdate date NOT NULL OPTIONS (ANNOTATION 'Start date for table.'),\n" +
" IN enddate date OPTIONS (ANNOTATION 'End date for table. If NULL, uses current date.')\n" +
" )\n" +
" RETURNS \n" +
" (\n" +
" \"month_start\" date\n" +
" ) AS\n" +
" BEGIN\n" +
" select CURDATE() as month_start;\n" +
" END;"
+ "create foreign procedure logMsg (level string, context string, msg string) returns boolean;"
+ "create foreign table address (addressid integer, city string);"
+ "create foreign table customeraddress (addressid integer);";

TransformationMetadata metadata = RealMetadataFactory.fromDDL(ddl, "x", "y");
CommandContext cc = createCommandContext();
ProcessorPlan plan = helpGetPlan(helpParse(sql), metadata, new DefaultCapabilitiesFinder(TestOptimizer.getTypicalCapabilities()), cc);

HardcodedDataManager dataManager = new HardcodedDataManager();
dataManager.addData("SELECT g_0.city FROM y.address AS g_0, y.customeraddress AS g_1 WHERE g_1.addressid = g_0.addressid", Arrays.asList("baltimore"));
dataManager.addData("EXEC logMsg('INFO', 'DEBUG.FOO.BAR', 'baltimore')", Arrays.asList(Boolean.TRUE));
helpProcess(plan, cc, dataManager, new List[] {Arrays.asList(Boolean.TRUE)});
}

}

73 changes: 73 additions & 0 deletions runtime/src/test/java/org/teiid/runtime/TestEmbeddedServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2568,5 +2568,78 @@ public void loadMetadata(MetadataFactory factory,
org.teiid.logging.LogManager.setLogListener(old);
}
}

@Test public void testSomething() throws Exception {
es.start(new EmbeddedConfiguration());
String sql = "SELECT \n" +
"(exec \"SYSADMIN.logMsg\"(\n" +
" \"level\" => 'INFO',\n" +
" \"context\" => 'DEBUG.FOO.BAR',\n" +
" \"msg\" => tccd.bank_account_holder_name)) as something\n" +
"FROM \n" +
"(\n" +
" SELECT c.city AS \"bank_account_holder_name\"\n" +
" FROM \"address\" c\n" +
" JOIN \n" +
" (\n" +
" select ca.addressid\n" +
" from \"customeraddress\" ca\n" +
" cross JOIN \n" +
" (\n" +
" WITH latest_exchange_rates AS \n" +
" (\n" +
" SELECT exchange_rate_date AS month_begin\n" +
" FROM (\n" +
" SELECT CURDATE() as exchange_rate_date\n" +
" )t\n" +
" )\n" +
" SELECT DISTINCT dt.month_start AS month_begin\n" +
" FROM (\n" +
" CALL createDateDimensionsTable(\n" +
" \"startdate\" => TIMESTAMPADD(SQL_TSI_MONTH, 1, (SELECT month_begin FROM latest_exchange_rates)),\n" +
" \"enddate\" => TIMESTAMPADD(SQL_TSI_YEAR, 15, (SELECT month_begin FROM latest_exchange_rates))\n" +
" ) ) AS dt\n" +
" ) fx\n" +
" ) ci ON ci.addressid = c.addressid\n" +
") tccd ;";

String ddl = "CREATE PROCEDURE createDateDimensionsTable(\n" +
" IN startdate date NOT NULL OPTIONS (ANNOTATION 'Start date for table.'),\n" +
" IN enddate date OPTIONS (ANNOTATION 'End date for table. If NULL, uses current date.')\n" +
" )\n" +
" RETURNS \n" +
" (\n" +
" \"month_start\" date\n" +
" ) AS\n" +
" BEGIN\n" +
" select CURDATE() as month_start;\n" +
" END;"
+ "create foreign table address (addressid integer, city string);"
+ "create foreign table customeraddress (addressid integer);";

ModelMetaData mmd = new ModelMetaData();
mmd.setName("y");
mmd.addSourceMetadata("ddl", ddl);
mmd.addSourceMapping("y", "y", null);
HardCodedExecutionFactory hcef = new HardCodedExecutionFactory();
hcef.addData("SELECT address.addressid, address.city FROM address", Arrays.asList(Arrays.asList(1, "baltimore")));
hcef.addData("SELECT customeraddress.addressid FROM customeraddress", Arrays.asList(Arrays.asList(1)));
es.addTranslator("y", hcef);

es.deployVDB("x", mmd);

Connection c = es.getDriver().connect("jdbc:teiid:x;", null);
Statement s = c.createStatement();
s.execute("set showplan debug");
try {
s.executeQuery(sql);
} finally {
ResultSet rs = s.executeQuery("show plan");
rs.next();
System.out.println(rs.getString(1));
System.out.println(rs.getString(2));
System.out.println(rs.getString(3));
}
}

}

0 comments on commit 8e9c726

Please sign in to comment.