From b061568ed880f40eaec1bc381170e7c5e9abf0f7 Mon Sep 17 00:00:00 2001 From: Chris Westin Date: Mon, 3 Aug 2015 13:13:21 -0700 Subject: [PATCH] DRILL-3594: don't use first query's allocator for rest of drillbit's life --- .../exec/planner/logical/DrillRuleSets.java | 102 +++++++++--------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java index 3f494988782..43077577524 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java @@ -62,9 +62,7 @@ import com.google.common.collect.ImmutableSet.Builder; public class DrillRuleSets { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRuleSets.class); - - public static RuleSet DRILL_BASIC_RULES = null; + //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRuleSets.class); /** * Get a list of logical rules that can be turned on or off by session/system options. @@ -80,11 +78,11 @@ public class DrillRuleSets { * rules that have been turned off by system or session settings */ public static RuleSet getDrillUserConfigurableLogicalRules(OptimizerRulesContext optimizerRulesContext) { - PlannerSettings ps = optimizerRulesContext.getPlannerSettings(); + final PlannerSettings ps = optimizerRulesContext.getPlannerSettings(); // This list is used to store rules that can be turned on an off // by user facing planning options - Builder userConfigurableRules = ImmutableSet.builder(); + final Builder userConfigurableRules = ImmutableSet.builder(); if (ps.isConstantFoldingEnabled()) { // TODO - DRILL-2218 @@ -97,32 +95,11 @@ public static RuleSet getDrillUserConfigurableLogicalRules(OptimizerRulesContext return new DrillRuleSet(userConfigurableRules.build()); } - /** - * Get an immutable list of rules that will always be used when running - * logical planning. - * - * This would be a static member, rather than a method, but some of - * the rules need a reference to state that isn't available at class - * load time. The current example is the DrillMergeProjectRule which - * needs access to the registry of Drill UDFs, which is populated by - * scanning the class path a Drillbit startup. - * - * If a logical rule needs to be user configurable, such as turning - * it on and off with a system/session option, add it in the - * getDrillUserConfigurableLogicalRules() method instead of here. - * - * @param optimizerRulesContext - shared state used during planning, currently used here - * to gain access to the function registry described above. - * @return - a RuleSet containing the logical rules that will always - * be used, either by VolcanoPlanner directly, or - * used VolcanoPlanner as pre-processing for LOPTPlanner. - * - * Note : Join permutation rule is excluded here. + /* + * These basic rules don't require any context, so singleton instances can be used. + * These are merged with per-query rules in getDrillBasicRules() below. */ - public static RuleSet getDrillBasicRules(OptimizerRulesContext optimizerRulesContext) { - if (DRILL_BASIC_RULES == null) { - - DRILL_BASIC_RULES = new DrillRuleSet(ImmutableSet. builder().add( // + private final static ImmutableSet staticRuleSet = ImmutableSet.builder().add( // Add support for Distinct Union (by using Union-All followed by Distinct) UnionToDistinctRule.INSTANCE, @@ -142,8 +119,6 @@ public static RuleSet getDrillBasicRules(OptimizerRulesContext optimizerRulesCon ProjectRemoveRule.NAME_CALC_INSTANCE, SortRemoveRule.INSTANCE, - DrillMergeProjectRule.getInstance(true, RelFactories.DEFAULT_PROJECT_FACTORY, - optimizerRulesContext.getFunctionRegistry()), AggregateExpandDistinctAggregatesRule.INSTANCE, DrillReduceAggregatesRule.INSTANCE, @@ -155,11 +130,6 @@ public static RuleSet getDrillBasicRules(OptimizerRulesContext optimizerRulesCon DrillPushProjIntoScan.INSTANCE, DrillProjectSetOpTransposeRule.INSTANCE, - PruneScanRule.getFilterOnProject(optimizerRulesContext), - PruneScanRule.getFilterOnScan(optimizerRulesContext), - ParquetPruneScanRule.getFilterOnProjectParquet(optimizerRulesContext), - ParquetPruneScanRule.getFilterOnScanParquet(optimizerRulesContext), - /* Convert from Calcite Logical to Drill Logical Rules. */ @@ -175,11 +145,46 @@ public static RuleSet getDrillBasicRules(OptimizerRulesContext optimizerRulesCon DrillJoinRule.INSTANCE, DrillUnionAllRule.INSTANCE, DrillValuesRule.INSTANCE - ) - .build()); - } + ).build(); - return DRILL_BASIC_RULES; + /** + * Get an immutable list of rules that will always be used when running + * logical planning. + * + * This cannot be a static singleton because some of the rules need to + * reference state owned by the current query (including its allocator). + * + * If a logical rule needs to be user configurable, such as turning + * it on and off with a system/session option, add it in the + * getDrillUserConfigurableLogicalRules() method instead of here. + * + * @param optimizerRulesContext - shared state used during planning, currently used here + * to gain access to the function registry described above. + * @return - a RuleSet containing the logical rules that will always + * be used, either by VolcanoPlanner directly, or + * used VolcanoPlanner as pre-processing for LOPTPlanner. + * + * Note : Join permutation rule is excluded here. + */ + public static RuleSet getDrillBasicRules(OptimizerRulesContext optimizerRulesContext) { + /* + * We have to create another copy of the ruleset with the context dependent elements; + * this cannot be reused across queries. + */ + final ImmutableSet basicRules = ImmutableSet.builder() + .addAll(staticRuleSet) + .add( + DrillMergeProjectRule.getInstance(true, RelFactories.DEFAULT_PROJECT_FACTORY, + optimizerRulesContext.getFunctionRegistry()), + + PruneScanRule.getFilterOnProject(optimizerRulesContext), + PruneScanRule.getFilterOnScan(optimizerRulesContext), + ParquetPruneScanRule.getFilterOnProjectParquet(optimizerRulesContext), + ParquetPruneScanRule.getFilterOnScanParquet(optimizerRulesContext) + ) + .build(); + + return new DrillRuleSet(basicRules); } // Ruleset for join permutation, used only in VolcanoPlanner. @@ -190,15 +195,13 @@ public static RuleSet getJoinPermRules(OptimizerRulesContext optimizerRulesConte ).build()); } - public static final RuleSet DRILL_PHYSICAL_DISK = new DrillRuleSet(ImmutableSet.of( // + public static final RuleSet DRILL_PHYSICAL_DISK = new DrillRuleSet(ImmutableSet.of( ProjectPrule.INSTANCE - )); public static final RuleSet getPhysicalRules(OptimizerRulesContext optimizerRulesContext) { - List ruleList = new ArrayList(); - - PlannerSettings ps = optimizerRulesContext.getPlannerSettings(); + final List ruleList = new ArrayList(); + final PlannerSettings ps = optimizerRulesContext.getPlannerSettings(); ruleList.add(ConvertCountToDirectScan.AGG_ON_PROJ_ON_SCAN); ruleList.add(ConvertCountToDirectScan.AGG_ON_SCAN); @@ -255,9 +258,9 @@ public static RuleSet create(ImmutableSet rules) { } public static RuleSet mergedRuleSets(RuleSet...ruleSets) { - Builder relOptRuleSetBuilder = ImmutableSet.builder(); - for (RuleSet ruleSet : ruleSets) { - for (RelOptRule relOptRule : ruleSet) { + final Builder relOptRuleSetBuilder = ImmutableSet.builder(); + for (final RuleSet ruleSet : ruleSets) { + for (final RelOptRule relOptRule : ruleSet) { relOptRuleSetBuilder.add(relOptRule); } } @@ -268,7 +271,6 @@ private static class DrillRuleSet implements RuleSet{ final ImmutableSet rules; public DrillRuleSet(ImmutableSet rules) { - super(); this.rules = rules; }