Skip to content

HIVE-27238: Avoid Calcite Code generation for MetadataHandler on ever… #4212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

HIVE-27238: Avoid Calcite Code generation for MetadataHandler on ever… #4212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a16e572
HIVE-27238: Avoid Calcite Code generation for MetadataHandler on ever…
scarlin-cloudera Apr 4, 2023
ae3c051
fixed code smells
scarlin-cloudera Apr 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@
package org.apache.hadoop.hive.ql.optimizer.calcite;

import java.util.List;
import java.util.Map;

import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
import org.apache.commons.collections4.map.LRUMap;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveDefaultCostModel;
import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveOnTezCostModel;
Expand All @@ -43,9 +45,18 @@
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdUniqueKeys;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;

public class HiveDefaultRelMetadataProvider {

// Set the number of providers allowed to be cached. A RelMetadataProvider provides Calcite
// generated code. Each instance generates new code so we want to limit the amount stored.
// The key to the provider contains various configuration variables since they are used in
// the calculations. These will most likely rarely be changed.
private static final int MAX_PROVIDERS = 20;
private static Map<Map<HiveConf.ConfVars, Object>, HiveDefaultRelMetadataProvider> ALL_PROVIDERS =
new LRUMap<>(MAX_PROVIDERS);

/**
* The default metadata provider can be instantiated statically since
* it does not need any parameter specified by user (hive conf).
Expand Down Expand Up @@ -128,4 +139,28 @@ public static void initializeMetadataProviderClass(List<Class<? extends RelNode>
// This will register the classes in the default Hive implementation
DEFAULT.register(nodeClasses);
}

public static synchronized HiveDefaultRelMetadataProvider get(HiveConf hiveConf,
List<Class<? extends RelNode>> nodeClasses) {
Map<HiveConf.ConfVars, Object> confKey = getConfKey(hiveConf);
if (ALL_PROVIDERS.containsKey(confKey)) {
return ALL_PROVIDERS.get(confKey);
}

HiveDefaultRelMetadataProvider newProvider =
new HiveDefaultRelMetadataProvider(hiveConf, nodeClasses);
ALL_PROVIDERS.put(confKey, newProvider);
return newProvider;
}

private static Map<HiveConf.ConfVars, Object> getConfKey(HiveConf conf) {
ImmutableMap.Builder<HiveConf.ConfVars, Object> bldr = new ImmutableMap.Builder<>();
bldr.put(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE,
conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE));
bldr.put(HiveConf.ConfVars.HIVE_CBO_EXTENDED_COST_MODEL,
conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_EXTENDED_COST_MODEL));
bldr.put(HiveConf.ConfVars.MAPREDMAXSPLITSIZE,
conf.getLongVar(HiveConf.ConfVars.MAPREDMAXSPLITSIZE));
Comment on lines +158 to +163
Copy link
Contributor

@kasakrisz kasakrisz Apr 25, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are these settings can be changed without hs2 restart?

return bldr.build();
}
}
9 changes: 3 additions & 6 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,6 @@
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.SetOp;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
Expand Down Expand Up @@ -1667,7 +1665,8 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu
calcitePlan.getCluster().getPlanner().setExecutor(executorProvider);

// Create and set MD provider
HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf, HIVE_REL_NODE_CLASSES);
HiveDefaultRelMetadataProvider mdProvider =
HiveDefaultRelMetadataProvider.get(conf, HIVE_REL_NODE_CLASSES);
RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider.getMetadataProvider()));
optCluster.invalidateMetadataQuery();

Expand Down Expand Up @@ -2438,9 +2437,7 @@ protected RelNode executeProgram(RelNode basePlan, HepProgram program,
List<RelMetadataProvider> list = Lists.newArrayList();
list.add(mdProvider);
planner.registerMetadataProviders(list);
RelMetadataProvider chainedProvider = ChainedRelMetadataProvider.of(list);
cluster.setMetadataProvider(
new CachingRelMetadataProvider(chainedProvider, planner));
cluster.setMetadataProvider(mdProvider);

if (executorProvider != null) {
// basePlan.getCluster.getPlanner is the VolcanoPlanner from apply()
Expand Down