Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Fixed AggregatePlanNodes for when we're only pushing down Distincts. …

…Still not quite done yet...
  • Loading branch information...
commit d6c41e0e1246947abd3d16ba308f222e61819cca 1 parent 31eff68
@apavlo authored
View
26 src/frontend/edu/brown/optimizer/optimizations/AggregatePushdownOptimization.java
@@ -158,18 +158,20 @@ public AggregatePushdownOptimization(PlanOptimizerState state) {
state.markDirty(send_node);
// 2011-12-08: We now need to correct the aggregate columns for the original plan node
- node.getAggregateColumnGuids().clear();
- for (Integer aggOutput : clone_node.getOutputColumnGUIDs()) {
- PlanColumn planCol = state.plannerContext.get(aggOutput);
- assert(planCol != null);
- AbstractExpression exp = planCol.getExpression();
- assert(exp != null);
- Collection<String> refTables = ExpressionUtil.getReferencedTableNames(exp);
- assert(refTables != null);
- if (refTables.size() == 1 && refTables.contains(PlanAssembler.AGGREGATE_TEMP_TABLE)) {
- node.getAggregateColumnGuids().add(planCol.guid());
- }
- } // FOR
+ if ((clone_node instanceof DistinctPlanNode) == false) {
+ node.getAggregateColumnGuids().clear();
+ for (Integer aggOutput : clone_node.getOutputColumnGUIDs()) {
+ PlanColumn planCol = state.plannerContext.get(aggOutput);
+ assert(planCol != null);
+ AbstractExpression exp = planCol.getExpression();
+ assert(exp != null);
+ Collection<String> refTables = ExpressionUtil.getReferencedTableNames(exp);
+ assert(refTables != null);
+ if (refTables.size() == 1 && refTables.contains(PlanAssembler.AGGREGATE_TEMP_TABLE)) {
+ node.getAggregateColumnGuids().add(planCol.guid());
+ }
+ } // FOR
+ }
if (debug.get()) {
LOG.debug("Successfully applied optimization! Eat that John Hugg!");
View
52 src/frontend/org/voltdb/compiler/StatementCompiler.java
@@ -145,22 +145,24 @@ else if (stmt.toLowerCase().startsWith("select")) {
QueryPlanner planner = new QueryPlanner(catalog.getClusters().get("cluster"), db, hsql, estimates, true, false);
- Exception first_exception = null;
- for (boolean _singleSited : new Boolean[] { true, false }) {
+ Throwable first_exception = null;
+ for (boolean _singleSited : new boolean[]{ true, false }) {
QueryType stmt_type = QueryType.get(catalogStmt.getQuerytype());
- compiler.addInfo("Creating " + stmt_type.name() + " query plan for " + catalogStmt.fullName() + ": singleSited=" + _singleSited);
- // System.err.println("Creating " + stmt_type.name() + " query plan for " + catalogStmt.getName() + ": singleSited=" + _singleSited);
- catalogStmt.setSinglepartition(_singleSited);
+ String msg = "Creating " + stmt_type.name() + " query plan for " + catalogStmt.fullName() + ": singleSited=" + _singleSited;
+ if (trace.get())
+ LOG.trace(msg);
+ compiler.addInfo(msg);
+ catalogStmt.setSinglepartition(_singleSited);
String name = catalogStmt.getParent().getName() + "-" + catalogStmt.getName();
- //System.out.println("stmt: " + name);
TrivialCostModel costModel = new TrivialCostModel();
try {
plan = planner.compilePlan(costModel, catalogStmt.getSqltext(),
catalogStmt.getName(), catalogStmt.getParent().getName(),
catalogStmt.getSinglepartition(), null);
- } catch (Exception e) {
+ } catch (Throwable e) {
+ LOG.error("Failed to plan for stmt: " + catalogStmt.fullName(), e);
if (first_exception == null) {
if (debug.get()) LOG.warn("Ignoring first error for " + catalogStmt.getName() + " :: " + e.getMessage());
first_exception = e;
@@ -171,7 +173,7 @@ else if (stmt.toLowerCase().startsWith("select")) {
}
if (plan == null) {
- String msg = "Failed to plan for stmt: " + catalogStmt.fullName();
+ msg = "Failed to plan for stmt: " + catalogStmt.fullName();
String plannerMsg = planner.getErrorMessage();
if (plannerMsg == null) plannerMsg = "PlannerMessage was empty!";
@@ -179,22 +181,30 @@ else if (stmt.toLowerCase().startsWith("select")) {
// HACK: Ignore if they were trying to do a single-sited INSERT/UPDATE/DELETE
// on a replicated table
if (plannerMsg.contains("replicated table") && _singleSited) {
- if (debug.get()) LOG.warn("Ignoring error: " + plannerMsg);
+ if (debug.get())
+ LOG.warn(String.format("Ignoring error for %s: %s", catalogStmt.fullName(), plannerMsg));
continue;
// HACK: If we get an unknown error message on an multi-sited INSERT/UPDATE/DELETE, assume
// that it's because we are trying to insert on a non-replicated table
} else if (!_singleSited && stmt_type == QueryType.INSERT && plannerMsg.contains("Error unknown")) {
- if (debug.get()) LOG.warn("Ignoring multi-sited " + stmt_type.name() + " on non-replicated table: " + plannerMsg);
+ if (debug.get())
+ LOG.warn(String.format("Ignoring multi-sited %s %s on non-replicated table: %s",
+ stmt_type.name(), catalogStmt.fullName(), plannerMsg));
continue;
} else if (planner.getError() != null) {
+ if (debug.get()) LOG.error(msg);
throw compiler.new VoltCompilerException(msg, planner.getError());
// Otherwise, report the error
} else {
if (plannerMsg != null)
msg += " with error: \"" + plannerMsg + "\"";
+ if (debug.get()) LOG.error(msg);
throw compiler.new VoltCompilerException(msg);
}
}
+ if (trace.get())
+ LOG.trace(String.format("%s Analyzing %s query plan",
+ catalogStmt.fullName(), (_singleSited == false ? "DTXN" : "SP")));
// serialize full where clause to the catalog
// for the benefit of the designer
@@ -219,17 +229,6 @@ else if (stmt.toLowerCase().startsWith("select")) {
// for the benefit of the designer
if (plan.fullWinnerPlan != null) {
String json = plan.fullplan_json;
-// try {
-// // serialize to pretty printed json
-//// String jsonCompact = plan.fullWinnerPlan.toJSONString();
-// // pretty printing seems to cause issues
-// //JSONObject jobj = new JSONObject(jsonCompact);
-// //json = jobj.toString(4);
-//// json = jsonCompact;
-// } catch (Exception e) {
-// // hopefully someone will notice
-// e.printStackTrace();
-// }
String hexString = Encoder.hexEncode(json);
if (_singleSited) {
catalogStmt.setFullplan(hexString);
@@ -248,6 +247,8 @@ else if (stmt.toLowerCase().startsWith("select")) {
int i = 0;
Collections.sort(plan.fragments);
+ if (trace.get())
+ LOG.trace(catalogStmt.fullName() + " Plan Fragments: " + plan.fragments);
for (CompiledPlan.Fragment fragment : plan.fragments) {
node_list = new PlanNodeList(fragment.planGraph);
@@ -261,11 +262,13 @@ else if (stmt.toLowerCase().startsWith("select")) {
if (_singleSited) {
planFragment = catalogStmt.getFragments().add(planFragmentName);
catalogStmt.setHas_singlesited(true);
- // System.err.println("SS PLAN FRAGMENT: " + planFragment.getGuid());
+ if (trace.get())
+ LOG.trace(String.format("%s SP PLAN FRAGMENT: %s", catalogStmt.fullName(), planFragment));
} else {
planFragment = catalogStmt.getMs_fragments().add(planFragmentName);
catalogStmt.setHas_multisited(true);
- // System.err.println("MS PLAN FRAGMENT: " + planFragment.getGuid());
+ if (trace.get())
+ LOG.trace(String.format("%s DTXN PLAN FRAGMENT: %s", catalogStmt.fullName(), planFragment));
}
// mark a fragment as non-transactional if it never touches a persistent table
@@ -280,8 +283,7 @@ else if (stmt.toLowerCase().startsWith("select")) {
JSONObject jobj = new JSONObject(node_list.toJSONString());
json = jobj.toString(4);
} catch (JSONException e2) {
- e2.printStackTrace();
- System.exit(-1);
+ throw new RuntimeException(e2);
}
// TODO: can't re-enable this until the EE accepts PlanColumn GUIDs
View
14 tests/frontend/edu/brown/optimizer/BasePlanOptimizerTestCase.java
@@ -340,7 +340,7 @@ else if (element instanceof ProjectionPlanNode) {
public static void validate(final AbstractPlanNode node) throws Exception {
- System.err.println("Validating: " + node + " / " + node.getPlanNodeType());
+// System.err.println("Validating: " + node + " / " + node.getPlanNodeType());
switch (node.getPlanNodeType()) {
// Make sure that the output columns from this node match the output
@@ -372,9 +372,8 @@ public static void validate(final AbstractPlanNode node) throws Exception {
// Make sure the DISTINCT column is in the output columns
DistinctPlanNode cast_node = (DistinctPlanNode)node;
Integer distinct_col = cast_node.getDistinctColumnGuid();
- if (cast_node.getOutputColumnGUIDs().contains(distinct_col) == false) {
- throw new Exception(String.format("%s is missing DISTINCT PlanColumn GUID %d in its output columns", cast_node, distinct_col));
- }
+ assertTrue(String.format("%s is missing DISTINCT PlanColumn GUID %d in its output columns", cast_node, distinct_col),
+ cast_node.getOutputColumnGUIDs().contains(distinct_col));
break;
}
@@ -384,7 +383,12 @@ public static void validate(final AbstractPlanNode node) throws Exception {
Collection<Integer> planCols = node.getOutputColumnGUIDs();
assert(planCols != null);
- System.err.println("PLAN COLS: " + planCols);
+// System.err.println(PlanNodeUtil.debugNode(node));
+ AggregatePlanNode cast_node = (AggregatePlanNode)node;
+ assertEquals(cast_node.toString(), cast_node.getAggregateTypes().size(), cast_node.getAggregateColumnGuids().size());
+ assertEquals(cast_node.toString(), cast_node.getAggregateTypes().size(), cast_node.getAggregateColumnNames().size());
+ assertEquals(cast_node.toString(), cast_node.getAggregateTypes().size(), cast_node.getAggregateOutputColumns().size());
+
// Set<Integer> foundCols = new HashSet<Integer>();
// for (AbstractPlanNode child : node.getChildren()) {
View
50 tests/frontend/edu/brown/optimizer/TestPlanOptimizer.java
@@ -1,7 +1,9 @@
package edu.brown.optimizer;
import java.util.Collection;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.Set;
import org.junit.Test;
import org.voltdb.catalog.Column;
@@ -32,6 +34,11 @@
*/
public class TestPlanOptimizer extends BasePlanOptimizerTestCase {
+ final Set<String> DEBUG = new HashSet<String>();
+ {
+ DEBUG.add("DistinctAggregate");
+ }
+
AbstractProjectBuilder pb = new PlanOptimizerTestProjectBuilder("planopt") {
{
this.addStmtProcedure("DistinctAggregate",
@@ -81,9 +88,6 @@
this.addStmtProcedure("OrderBy",
"SELECT TABLEC.C_B_A_ID FROM TABLEC ORDER BY TABLEC.C_B_A_ID, TABLEC.C_VALUE0");
-// this.addStmtProcedure("GroupBy",
-// "SELECT MAX(TABLEC.C_ID) FROM TABLEC GROUP BY TABLEC.C_B_A_ID, TABLEC.C_VALUE0");
-
this.addStmtProcedure("LimitOrderBy",
"SELECT C_ID FROM TABLEC ORDER BY C_B_A_ID LIMIT 1000");
@@ -109,49 +113,13 @@ private void check(Statement catalog_stmt) throws Exception {
for (boolean dtxn : new boolean[]{ true, false }) {
AbstractPlanNode root = PlanNodeUtil.getRootPlanNodeForStatement(catalog_stmt, dtxn);
assertNotNull(root);
-// System.err.println(PlanNodeUtil.debug(root));
+ if (DEBUG.contains(catalog_stmt.getName()))
+ System.err.println(PlanNodeUtil.debug(root));
BasePlanOptimizerTestCase.validate(root);
} // FOR
}
/**
- * testExtractColumnInfo
- */
-// @Test
-// public void testExtractColumnInfo() throws Exception {
-// Procedure catalog_proc = this.getProcedure("DistinctCount");
-// Statement catalog_stmt = this.getStatement(catalog_proc, "sql");
-//
-// AbstractPlanNode root = PlanNodeUtil.getRootPlanNodeForStatement(catalog_stmt, true);
-// assertNotNull(root);
-// PlanOptimizerState state = new PlanOptimizerState(catalog_db, PlannerContext.singleton());
-//
-// Collection<SeqScanPlanNode> scan_nodes = PlanNodeUtil.getPlanNodes(root, SeqScanPlanNode.class);
-// SeqScanPlanNode scan_node = CollectionUtil.first(scan_nodes);
-// assertNotNull(scan_node);
-//
-// // Use the PlanOptimizerUtil to compute the referenced columns for this node
-// PlanOptimizerUtil.populateTableNodeInfo(state, root);
-//
-// // Then make sure it has the right references
-// Table catalog_tbl = this.getTable(scan_node.getTargetTableName());
-// Set<Column> expected = new HashSet<Column>();
-// for (Integer guid : scan_node.getOutputColumnGUIDs()) {
-// PlanColumn pc = state.plannerContext.get(guid);
-// assertNotNull(pc);
-//
-// Column catalog_col = catalog_tbl.getColumns().getIgnoreCase(pc.getDisplayName());
-// assertNotNull(pc.toString(), catalog_col);
-// expected.add(catalog_col);
-// } // FOR
-//
-// Collection<Column> actual = state.getPlanNodeColumns(scan_node);
-// assertNotNull(actual);
-// assertEquals(expected.size(), actual.size());
-// assertTrue(expected.containsAll(actual));
-// }
-
- /**
* testExtractReferencedColumns
*/
@Test
View
38 tests/frontend/edu/brown/optimizer/TestPlanOptimizerUtil.java
@@ -157,5 +157,43 @@ protected void callback(AbstractPlanNode element) {
System.out.println("guid_column_xref: " + state.guid_column_xref);
System.out.println("orig_node_output: " + state.orig_node_output);
}
+
+
+ /**
+ * testExtractColumnInfo
+ */
+// @Test
+// public void testExtractColumnInfo() throws Exception {
+// Procedure catalog_proc = this.getProcedure("DistinctCount");
+// Statement catalog_stmt = this.getStatement(catalog_proc, "sql");
+//
+// AbstractPlanNode root = PlanNodeUtil.getRootPlanNodeForStatement(catalog_stmt, true);
+// assertNotNull(root);
+// PlanOptimizerState state = new PlanOptimizerState(catalog_db, PlannerContext.singleton());
+//
+// Collection<SeqScanPlanNode> scan_nodes = PlanNodeUtil.getPlanNodes(root, SeqScanPlanNode.class);
+// SeqScanPlanNode scan_node = CollectionUtil.first(scan_nodes);
+// assertNotNull(scan_node);
+//
+// // Use the PlanOptimizerUtil to compute the referenced columns for this node
+// PlanOptimizerUtil.populateTableNodeInfo(state, root);
+//
+// // Then make sure it has the right references
+// Table catalog_tbl = this.getTable(scan_node.getTargetTableName());
+// Set<Column> expected = new HashSet<Column>();
+// for (Integer guid : scan_node.getOutputColumnGUIDs()) {
+// PlanColumn pc = state.plannerContext.get(guid);
+// assertNotNull(pc);
+//
+// Column catalog_col = catalog_tbl.getColumns().getIgnoreCase(pc.getDisplayName());
+// assertNotNull(pc.toString(), catalog_col);
+// expected.add(catalog_col);
+// } // FOR
+//
+// Collection<Column> actual = state.getPlanNodeColumns(scan_node);
+// assertNotNull(actual);
+// assertEquals(expected.size(), actual.size());
+// assertTrue(expected.containsAll(actual));
+// }
}
Please sign in to comment.
Something went wrong with that request. Please try again.