diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java index 5b6570a1bad2..fca71af1c519 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java @@ -18,11 +18,13 @@ package org.apache.hadoop.hive.ql.optimizer.calcite; import java.util.List; +import java.util.Map; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; +import org.apache.commons.collections4.map.LRUMap; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveDefaultCostModel; import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveOnTezCostModel; @@ -43,9 +45,18 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdUniqueKeys; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; public class HiveDefaultRelMetadataProvider { + // Set the number of providers allowed to be cached. A RelMetadataProvider provides Calcite + // generated code. Each instance generates new code so we want to limit the amount stored. + // The key to the provider contains various configuration variables since they are used in + // the calculations. These will most likely rarely be changed. + private static final int MAX_PROVIDERS = 20; + private static Map, HiveDefaultRelMetadataProvider> ALL_PROVIDERS = + new LRUMap<>(MAX_PROVIDERS); + /** * The default metadata provider can be instantiated statically since * it does not need any parameter specified by user (hive conf). @@ -128,4 +139,28 @@ public static void initializeMetadataProviderClass(List // This will register the classes in the default Hive implementation DEFAULT.register(nodeClasses); } + + public static synchronized HiveDefaultRelMetadataProvider get(HiveConf hiveConf, + List> nodeClasses) { + Map confKey = getConfKey(hiveConf); + if (ALL_PROVIDERS.containsKey(confKey)) { + return ALL_PROVIDERS.get(confKey); + } + + HiveDefaultRelMetadataProvider newProvider = + new HiveDefaultRelMetadataProvider(hiveConf, nodeClasses); + ALL_PROVIDERS.put(confKey, newProvider); + return newProvider; + } + + private static Map getConfKey(HiveConf conf) { + ImmutableMap.Builder bldr = new ImmutableMap.Builder<>(); + bldr.put(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, + conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE)); + bldr.put(HiveConf.ConfVars.HIVE_CBO_EXTENDED_COST_MODEL, + conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_EXTENDED_COST_MODEL)); + bldr.put(HiveConf.ConfVars.MAPREDMAXSPLITSIZE, + conf.getLongVar(HiveConf.ConfVars.MAPREDMAXSPLITSIZE)); + return bldr.build(); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index d8850ffbafb2..3ea0491a23a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -84,8 +84,6 @@ import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.metadata.CachingRelMetadataProvider; -import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataQuery; @@ -1667,7 +1665,8 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu calcitePlan.getCluster().getPlanner().setExecutor(executorProvider); // Create and set MD provider - HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf, HIVE_REL_NODE_CLASSES); + HiveDefaultRelMetadataProvider mdProvider = + HiveDefaultRelMetadataProvider.get(conf, HIVE_REL_NODE_CLASSES); RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider.getMetadataProvider())); optCluster.invalidateMetadataQuery(); @@ -2438,9 +2437,7 @@ protected RelNode executeProgram(RelNode basePlan, HepProgram program, List list = Lists.newArrayList(); list.add(mdProvider); planner.registerMetadataProviders(list); - RelMetadataProvider chainedProvider = ChainedRelMetadataProvider.of(list); - cluster.setMetadataProvider( - new CachingRelMetadataProvider(chainedProvider, planner)); + cluster.setMetadataProvider(mdProvider); if (executorProvider != null) { // basePlan.getCluster.getPlanner is the VolcanoPlanner from apply()