From e0c9b841d4001c2574cf015a91974c536528d345 Mon Sep 17 00:00:00 2001 From: Jinfeng Ni Date: Tue, 3 Nov 2015 15:51:10 -0800 Subject: [PATCH] DRILL-3765: Move partitioning pruning to HepPlanner to avoid the performance overhead for redundant rule execution. Add fall back option in planner. close apache/drill#255 --- .../HivePushPartitionFilterIntoScan.java | 13 ++- .../exec/planner/common/DrillJoinRelBase.java | 2 +- .../exec/planner/logical/DrillRuleSets.java | 27 +++++- .../exec/planner/logical/DrillScanRel.java | 32 ++++++- .../partition/ParquetPruneScanRule.java | 13 ++- .../logical/partition/PruneScanRule.java | 15 +++- .../planner/physical/PlannerSettings.java | 7 +- .../exec/planner/sql/DrillSqlWorker.java | 27 ++++-- .../sql/handlers/DefaultSqlHandler.java | 88 ++++++++++--------- .../server/options/SystemOptionManager.java | 3 +- 10 files changed, 159 insertions(+), 68 deletions(-) diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java index 105a07e799e..fc2007eaffc 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java @@ -27,6 +27,7 @@ import org.apache.drill.exec.planner.logical.RelOptHelper; import org.apache.drill.exec.planner.logical.partition.PruneScanRule; import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.planner.physical.PrelUtil; import org.apache.drill.exec.planner.sql.HivePartitionDescriptor; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import org.apache.drill.exec.store.hive.HiveScan; @@ -52,7 +53,11 @@ public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(2); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for Hive based partition pruning - return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown(); + if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { + return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); + } else { + return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown(); + } } @Override @@ -82,7 +87,11 @@ public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(1); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for Hive based partition pruning - return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown(); + if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { + return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); + } else { + return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown(); + } } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java index 7d3ad0acbf1..2d6f7d6a393 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java @@ -74,7 +74,7 @@ public RelOptCost computeSelfCost(RelOptPlanner planner) { * - Such filter on top of join might be pushed into JOIN, when LOPT planner is called. * - Return non-infinite cost will give LOPT planner a chance to try to push the filters. */ - if (PrelUtil.getPlannerSettings(planner).isHepJoinOptEnabled()) { + if (PrelUtil.getPlannerSettings(planner).isHepOptEnabled()) { return computeCartesianJoinCost(planner); } else { return ((DrillCostFactory)planner.getCostFactory()).makeInfiniteCost(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java index 16ec559e2b9..b7974f39ed7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java @@ -31,12 +31,15 @@ import org.apache.calcite.rel.rules.FilterSetOpTransposeRule; import org.apache.calcite.rel.rules.JoinPushExpressionsRule; import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; +import org.apache.calcite.rel.rules.JoinToMultiJoinRule; +import org.apache.calcite.rel.rules.LoptOptimizeJoinRule; import org.apache.calcite.rel.rules.ProjectRemoveRule; import org.apache.calcite.rel.rules.ProjectWindowTransposeRule; import org.apache.calcite.rel.rules.ReduceExpressionsRule; import org.apache.calcite.rel.rules.SortRemoveRule; import org.apache.calcite.rel.rules.UnionToDistinctRule; import org.apache.calcite.tools.RuleSet; +import org.apache.commons.digester.Rules; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.planner.logical.partition.ParquetPruneScanRule; import org.apache.drill.exec.planner.logical.partition.PruneScanRule; @@ -67,6 +70,12 @@ public class DrillRuleSets { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRuleSets.class); + public static final RelOptRule DRILL_JOIN_TO_MULTIJOIN_RULE = new JoinToMultiJoinRule(DrillJoinRel.class); + public static final RelOptRule DRILL_LOPT_OPTIMIZE_JOIN_RULE = new LoptOptimizeJoinRule( + DrillRelFactories.DRILL_LOGICAL_JOIN_FACTORY, + DrillRelFactories.DRILL_LOGICAL_PROJECT_FACTORY, + DrillRelFactories.DRILL_LOGICAL_FILTER_FACTORY); + /** * Get a list of logical rules that can be turned on or off by session/system options. * @@ -90,7 +99,6 @@ public static RuleSet getDrillUserConfigurableLogicalRules(OptimizerRulesContext if (ps.isConstantFoldingEnabled()) { // TODO - DRILL-2218 userConfigurableRules.add(ReduceExpressionsRule.PROJECT_INSTANCE); - userConfigurableRules.add(DrillReduceExpressionsRule.FILTER_INSTANCE_DRILL); userConfigurableRules.add(DrillReduceExpressionsRule.CALC_INSTANCE_DRILL); } @@ -183,16 +191,27 @@ public static RuleSet getDrillBasicRules(OptimizerRulesContext optimizerRulesCon .addAll(staticRuleSet) .add( DrillMergeProjectRule.getInstance(true, RelFactories.DEFAULT_PROJECT_FACTORY, - optimizerRulesContext.getFunctionRegistry()), + optimizerRulesContext.getFunctionRegistry()) + ) + .build(); + + return new DrillRuleSet(basicRules); + } + /** + * Get an immutable list of partition pruning rules that will be used in logical planning. + */ + public static RuleSet getPruneScanRules(OptimizerRulesContext optimizerRulesContext) { + final ImmutableSet pruneRules = ImmutableSet.builder() + .add( PruneScanRule.getFilterOnProject(optimizerRulesContext), PruneScanRule.getFilterOnScan(optimizerRulesContext), ParquetPruneScanRule.getFilterOnProjectParquet(optimizerRulesContext), ParquetPruneScanRule.getFilterOnScanParquet(optimizerRulesContext) - ) + ) .build(); - return new DrillRuleSet(basicRules); + return new DrillRuleSet(pruneRules); } // Ruleset for join permutation, used only in VolcanoPlanner. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java index 9a009ce96d5..0d087aa2d45 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java @@ -54,19 +54,31 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel { private GroupScan groupScan; private List columns; private PlannerSettings settings; + private final boolean partitionFilterPushdown; /** Creates a DrillScan. */ public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits, - final RelOptTable table) { + final RelOptTable table) { + this(cluster, traits, table, false); + } + /** Creates a DrillScan. */ + public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits, + final RelOptTable table, boolean partitionFilterPushdown) { // By default, scan does not support project pushdown. // Decision whether push projects into scan will be made solely in DrillPushProjIntoScanRule. - this(cluster, traits, table, table.getRowType(), GroupScan.ALL_COLUMNS); + this(cluster, traits, table, table.getRowType(), GroupScan.ALL_COLUMNS, partitionFilterPushdown); this.settings = PrelUtil.getPlannerSettings(cluster.getPlanner()); } /** Creates a DrillScan. */ public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits, - final RelOptTable table, final RelDataType rowType, final List columns) { + final RelOptTable table, final RelDataType rowType, final List columns) { + this(cluster, traits, table, rowType, columns, false); + } + + /** Creates a DrillScan. */ + public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits, + final RelOptTable table, final RelDataType rowType, final List columns, boolean partitionFilterPushdown) { super(DRILL_LOGICAL, cluster, traits, table); this.settings = PrelUtil.getPlannerSettings(cluster.getPlanner()); this.rowType = rowType; @@ -77,6 +89,7 @@ public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits, } else { // planner asks to scan some columns this.columns = ColumnList.some(columns); } + this.partitionFilterPushdown = partitionFilterPushdown; try { this.groupScan = drillTable.getGroupScan().clone(this.columns); } catch (final IOException e) { @@ -86,12 +99,19 @@ public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits, /** Creates a DrillScanRel for a particular GroupScan */ public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits, - final RelOptTable table, final GroupScan groupScan, final RelDataType rowType, final List columns) { + final RelOptTable table, final GroupScan groupScan, final RelDataType rowType, final List columns) { + this(cluster, traits, table, groupScan, rowType, columns, false); + } + + /** Creates a DrillScanRel for a particular GroupScan */ + public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits, + final RelOptTable table, final GroupScan groupScan, final RelDataType rowType, final List columns, boolean partitionFilterPushdown) { super(DRILL_LOGICAL, cluster, traits, table); this.rowType = rowType; this.columns = columns; this.groupScan = groupScan; this.settings = PrelUtil.getPlannerSettings(cluster.getPlanner()); + this.partitionFilterPushdown = partitionFilterPushdown; } // @@ -180,4 +200,8 @@ public GroupScan getGroupScan() { return groupScan; } + public boolean partitionFilterPushdown() { + return this.partitionFilterPushdown; + } + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java index a95185494e3..b4f4e95ff98 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java @@ -29,6 +29,7 @@ import org.apache.drill.exec.planner.logical.DrillScanRel; import org.apache.drill.exec.planner.logical.RelOptHelper; import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.planner.physical.PrelUtil; import org.apache.drill.exec.store.parquet.ParquetGroupScan; public class ParquetPruneScanRule { @@ -49,7 +50,11 @@ public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(2); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for parquet based partition pruning - return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown(); + if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { + return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); + } else { + return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown(); + } } @Override @@ -77,7 +82,11 @@ public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(1); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for parquet based partition pruning - return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown(); + if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { + return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); + } else { + return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown(); + } } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java index b7b58ec94e1..ab6c5a8a24f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java @@ -94,7 +94,11 @@ public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(2); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for dfs based partition pruning - return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown(); + if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { + return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); + } else { + return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown(); + } } @Override @@ -122,7 +126,11 @@ public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(1); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for dfs based partition pruning - return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown(); + if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { + return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); + } else { + return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown(); + } } @Override @@ -319,7 +327,8 @@ protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, DrillPro scanRel.getTable(), descriptor.createNewGroupScan(newFiles), scanRel.getRowType(), - scanRel.getColumns()); + scanRel.getColumns(), + true /*filter pushdown*/); RelNode inputRel = newScanRel; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java index 0a05d08f2e4..3eb50383071 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java @@ -72,7 +72,8 @@ public class PlannerSettings implements Context{ public static final OptionValidator HASH_JOIN_SWAP_MARGIN_FACTOR = new RangeDoubleValidator("planner.join.hash_join_swap_margin_factor", 0, 100, 10d); public static final String ENABLE_DECIMAL_DATA_TYPE_KEY = "planner.enable_decimal_data_type"; public static final OptionValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY, false); - public static final OptionValidator HEP_JOIN_OPT = new BooleanValidator("planner.enable_hep_join_opt", true); + public static final OptionValidator HEP_OPT = new BooleanValidator("planner.enable_hep_opt", true); + public static final OptionValidator HEP_PARTITION_PRUNING = new BooleanValidator("planner.enable_hep_partition_pruning", true); public static final OptionValidator PLANNER_MEMORY_LIMIT = new RangeLongValidator("planner.memory_limit", INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES, DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES); @@ -172,7 +173,9 @@ public boolean isHashJoinSwapEnabled() { return options.getOption(HASH_JOIN_SWAP.getOptionName()).bool_val; } - public boolean isHepJoinOptEnabled() { return options.getOption(HEP_JOIN_OPT.getOptionName()).bool_val;} + public boolean isHepPartitionPruningEnabled() { return options.getOption(HEP_PARTITION_PRUNING.getOptionName()).bool_val;} + + public boolean isHepOptEnabled() { return options.getOption(HEP_OPT.getOptionName()).bool_val;} public double getHashJoinSwapMarginFactor() { return options.getOption(HASH_JOIN_SWAP_MARGIN_FACTOR.getOptionName()).float_val / 100d; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index 37f0ecd8d10..92e59ec4f66 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -74,7 +74,8 @@ public class DrillSqlWorker { private final HepPlanner hepPlanner; public final static int LOGICAL_RULES = 0; public final static int PHYSICAL_MEM_RULES = 1; - public final static int LOGICAL_CONVERT_RULES = 2; + public final static int LOGICAL_HEP_JOIN_RULES = 2; + public final static int LOGICAL_HEP_JOIN__PP_RULES = 3; private final QueryContext context; @@ -116,20 +117,32 @@ public DrillSqlWorker(QueryContext context) { private RuleSet[] getRules(QueryContext context) { StoragePluginRegistry storagePluginRegistry = context.getStorage(); - RuleSet drillLogicalRules = DrillRuleSets.mergedRuleSets( + + // Ruleset for the case where VolcanoPlanner is used for everything : join, filter/project pushdown, partition pruning. + RuleSet drillLogicalVolOnlyRules = DrillRuleSets.mergedRuleSets( DrillRuleSets.getDrillBasicRules(context), + DrillRuleSets.getPruneScanRules(context), DrillRuleSets.getJoinPermRules(context), DrillRuleSets.getDrillUserConfigurableLogicalRules(context)); + + // Ruleset for the case where join planning is done in Hep-LOPT, filter/project pushdown and parttion pruning are done in VolcanoPlanner + RuleSet drillLogicalHepJoinRules = DrillRuleSets.mergedRuleSets( + DrillRuleSets.getDrillBasicRules(context), + DrillRuleSets.getPruneScanRules(context), + DrillRuleSets.getDrillUserConfigurableLogicalRules(context)); + + // Ruleset for the case where join planning and partition pruning is done in Hep, filter/project pushdown are done in VolcanoPlanner + RuleSet drillLogicalHepJoinPPRules = DrillRuleSets.mergedRuleSets( + DrillRuleSets.getDrillBasicRules(context), + DrillRuleSets.getDrillUserConfigurableLogicalRules(context)); + + // Ruleset for physical planning rules RuleSet drillPhysicalMem = DrillRuleSets.mergedRuleSets( DrillRuleSets.getPhysicalRules(context), storagePluginRegistry.getStoragePluginRuleSet(context)); - // Following is used in LOPT join OPT. - RuleSet logicalConvertRules = DrillRuleSets.mergedRuleSets( - DrillRuleSets.getDrillBasicRules(context), - DrillRuleSets.getDrillUserConfigurableLogicalRules(context)); - RuleSet[] allRules = new RuleSet[] {drillLogicalRules, drillPhysicalMem, logicalConvertRules}; + RuleSet[] allRules = new RuleSet[] {drillLogicalVolOnlyRules, drillPhysicalMem, drillLogicalHepJoinRules, drillLogicalHepJoinPPRules}; return allRules; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java index 9d11b960f46..686f7d7817f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java @@ -21,8 +21,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Set; +import com.google.common.collect.ImmutableSet; import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.plan.hep.HepMatchOrder; @@ -68,12 +71,17 @@ import org.apache.drill.exec.planner.common.DrillRelOptUtil; import org.apache.drill.exec.planner.cost.DrillDefaultRelMetadataProvider; import org.apache.drill.exec.planner.logical.DrillJoinRel; +import org.apache.drill.exec.planner.logical.DrillMergeProjectRule; import org.apache.drill.exec.planner.logical.DrillProjectRel; +import org.apache.drill.exec.planner.logical.DrillPushProjectPastFilterRule; import org.apache.drill.exec.planner.logical.DrillRel; import org.apache.drill.exec.planner.logical.DrillRelFactories; +import org.apache.drill.exec.planner.logical.DrillRuleSets; import org.apache.drill.exec.planner.logical.DrillScreenRel; import org.apache.drill.exec.planner.logical.DrillStoreRel; import org.apache.drill.exec.planner.logical.PreProcessLogicalRel; +import org.apache.drill.exec.planner.logical.partition.ParquetPruneScanRule; +import org.apache.drill.exec.planner.logical.partition.PruneScanRule; import org.apache.drill.exec.planner.physical.DrillDistributionTrait; import org.apache.drill.exec.planner.physical.PhysicalPlanCreator; import org.apache.drill.exec.planner.physical.PlannerSettings; @@ -207,11 +215,7 @@ protected DrillRel convertToDrel(RelNode relNode) throws SqlUnsupportedException try { final DrillRel convertedRelNode; - if (! context.getPlannerSettings().isHepJoinOptEnabled()) { - convertedRelNode = (DrillRel) logicalPlanningVolcano(relNode); - } else { - convertedRelNode = (DrillRel) logicalPlanningVolcanoAndLopt(relNode); - } + convertedRelNode = (DrillRel) doLogicalPlanning(relNode); if (convertedRelNode instanceof DrillStoreRel) { throw new UnsupportedOperationException(); @@ -529,55 +533,56 @@ protected DrillRel addRenamedProject(DrillRel rel, RelDataType validatedRowType) } - private RelNode logicalPlanningVolcano(RelNode relNode) throws RelConversionException, SqlUnsupportedException { - return planner.transform(DrillSqlWorker.LOGICAL_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL), relNode); - } + private RelNode doLogicalPlanning(RelNode relNode) throws RelConversionException, SqlUnsupportedException { + if (! context.getPlannerSettings().isHepOptEnabled()) { + return planner.transform(DrillSqlWorker.LOGICAL_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL), relNode); + } else { + RelNode convertedRelNode = null; + if (context.getPlannerSettings().isHepPartitionPruningEnabled()) { + convertedRelNode = planner.transform(DrillSqlWorker.LOGICAL_HEP_JOIN__PP_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL), relNode); + log("VolcanoRel", convertedRelNode, logger); - /** - * Do logical planning using both VolcanoPlanner and LOPT HepPlanner. - * @param relNode - * @return - * @throws RelConversionException - * @throws SqlUnsupportedException - */ - private RelNode logicalPlanningVolcanoAndLopt(RelNode relNode) throws RelConversionException, SqlUnsupportedException { + // Partition pruning . + ImmutableSet pruneScanRules = ImmutableSet.builder() + .addAll(DrillRuleSets.getPruneScanRules(context)) + .build(); - final RelNode convertedRelNode = planner.transform(DrillSqlWorker.LOGICAL_CONVERT_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL), relNode); - log("VolCalciteRel", convertedRelNode, logger); + convertedRelNode = doHepPlan(convertedRelNode, pruneScanRules, HepMatchOrder.BOTTOM_UP); + } else { + convertedRelNode = planner.transform(DrillSqlWorker.LOGICAL_HEP_JOIN_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL), relNode); + log("VolcanoRel", convertedRelNode, logger); + } - final RelNode loptNode = getLoptJoinOrderTree( - convertedRelNode, - DrillJoinRel.class, - DrillRelFactories.DRILL_LOGICAL_JOIN_FACTORY, - DrillRelFactories.DRILL_LOGICAL_FILTER_FACTORY, - DrillRelFactories.DRILL_LOGICAL_PROJECT_FACTORY); + // Join order planning with LOPT rule + ImmutableSet joinOrderRules = ImmutableSet.builder() + .add( + DrillRuleSets.DRILL_JOIN_TO_MULTIJOIN_RULE, + DrillRuleSets.DRILL_LOPT_OPTIMIZE_JOIN_RULE, + ProjectRemoveRule.INSTANCE) + .build(); - log("HepCalciteRel", loptNode, logger); + final RelNode loptNode = doHepPlan(convertedRelNode, joinOrderRules, HepMatchOrder.BOTTOM_UP); - return loptNode; - } + log("HepRel", loptNode, logger); + return loptNode; + } + } /** - * Appy Join Order Optimizations using Hep Planner. + * Use HepPlanner to apply optimization rules to RelNode tree. + * @return : the root node for optimized plan. */ - private RelNode getLoptJoinOrderTree(RelNode root, - Class joinClass, - RelFactories.JoinFactory joinFactory, - RelFactories.FilterFactory filterFactory, - RelFactories.ProjectFactory projectFactory) { + private RelNode doHepPlan(RelNode root, Set rules, HepMatchOrder matchOrder) { final HepProgramBuilder hepPgmBldr = new HepProgramBuilder() - .addMatchOrder(HepMatchOrder.BOTTOM_UP) - .addRuleInstance(new JoinToMultiJoinRule(joinClass)) - .addRuleInstance(new LoptOptimizeJoinRule(joinFactory, projectFactory, filterFactory)) - .addRuleInstance(ProjectRemoveRule.INSTANCE); - // .addRuleInstance(new ProjectMergeRule(true, projectFactory)); - - // .addRuleInstance(DrillMergeProjectRule.getInstance(true, projectFactory, this.context.getFunctionRegistry())); + .addMatchOrder(matchOrder); + for (final RelOptRule rule : rules) { + hepPgmBldr.addRuleInstance(rule); + } final HepProgram hepPgm = hepPgmBldr.build(); - final HepPlanner hepPlanner = new HepPlanner(hepPgm); + final HepPlanner hepPlanner = new HepPlanner(hepPgm, context.getPlannerSettings()); final List list = Lists.newArrayList(); list.add(DrillDefaultRelMetadataProvider.INSTANCE); @@ -594,7 +599,6 @@ private RelNode getLoptJoinOrderTree(RelNode root, return calciteOptimizedPlan; } - public static class MetaDataProviderModifier extends RelShuttleImpl { private final RelMetadataProvider metadataProvider; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 5e16483bb6f..72fcccba494 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -79,8 +79,9 @@ public class SystemOptionManager extends BaseOptionManager { PlannerSettings.PARTITION_SENDER_MAX_THREADS, PlannerSettings.PARTITION_SENDER_SET_THREADS, PlannerSettings.ENABLE_DECIMAL_DATA_TYPE, - PlannerSettings.HEP_JOIN_OPT, + PlannerSettings.HEP_OPT, PlannerSettings.PLANNER_MEMORY_LIMIT, + PlannerSettings.HEP_PARTITION_PRUNING, ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION, ExecConstants.OUTPUT_FORMAT_VALIDATOR, ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,