Skip to content

Commit

Permalink
HIVE-12679: PoC of Option 2
Browse files Browse the repository at this point in the history
  • Loading branch information
okumin committed Oct 1, 2023
1 parent 85f6162 commit d185af2
Show file tree
Hide file tree
Showing 3 changed files with 95 additions and 7 deletions.
8 changes: 1 addition & 7 deletions ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,6 @@
import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.PartitionDropOptions;
import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
import org.apache.hadoop.hive.metastore.SynchronizedMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
Expand Down Expand Up @@ -5766,12 +5765,7 @@ public HiveMetaHook getHook(
}
};

if (conf.getBoolVar(ConfVars.METASTORE_FASTPATH)) {
return new SessionHiveMetaStoreClient(conf, hookLoader, allowEmbedded);
} else {
return RetryingMetaStoreClient.getProxy(conf, hookLoader, metaCallTimeMap,
SessionHiveMetaStoreClient.class.getName(), allowEmbedded);
}
return HiveMetaStoreClientFactory.createMetaStoreClient(conf, hookLoader, allowEmbedded, metaCallTimeMap);
}

@Nullable
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package org.apache.hadoop.hive.ql.metadata;

import static com.google.common.base.Preconditions.checkNotNull;

import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.ExceptionHandler;
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;

public class HiveMetaStoreClientFactory {
public static IMetaStoreClient createMetaStoreClient(HiveConf conf, HiveMetaHookLoader hookLoader,
boolean allowEmbedded, ConcurrentHashMap<String, Long> metaCallTimeMap) throws MetaException {
checkNotNull(conf, "conf cannot be null!");
checkNotNull(hookLoader, "hookLoader cannot be null!");
checkNotNull(metaCallTimeMap, "metaCallTimeMap cannot be null!");

String clientClassName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METASTORE_CLIENT_CLASS);

if (clientClassName.equals(SessionHiveMetaStoreClient.class.getName())
&& conf.getBoolVar(ConfVars.METASTORE_FASTPATH)) {
// METASTORE_FASTPATH is a parameter only for SessionHiveMetaStoreClient
return new SessionHiveMetaStoreClient(conf, hookLoader, allowEmbedded);
}

String proxyClassName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METASTORE_CLIENT_PROXY_CLASS);
if (proxyClassName == null) {
// Without a proxy
Class<? extends IMetaStoreClient> baseClass = JavaUtils.getClass(clientClassName, IMetaStoreClient.class);
return JavaUtils.newInstance(
baseClass,
new Class[] { Configuration.class, HiveMetaHookLoader.class, Boolean.class },
new Object[] { conf, hookLoader, allowEmbedded });
}

return getProxy(conf, hookLoader, metaCallTimeMap, clientClassName, allowEmbedded, proxyClassName);
}

private static IMetaStoreClient getProxy(Configuration hiveConf, HiveMetaHookLoader hookLoader,
ConcurrentHashMap<String, Long> metaCallTimeMap, String mscClassName, boolean allowEmbedded,
String proxyClassName) throws MetaException {
return getProxy(hiveConf,
new Class[] {Configuration.class, HiveMetaHookLoader.class, Boolean.class},
new Object[] {hiveConf, hookLoader, allowEmbedded},
metaCallTimeMap,
mscClassName,
proxyClassName
);
}

private static IMetaStoreClient getProxy(Configuration hiveConf, Class<?>[] constructorArgTypes,
Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap,
String mscClassName, String proxyClassName) throws MetaException {

@SuppressWarnings("unchecked")
Class<? extends IMetaStoreClient> baseClass =
JavaUtils.getClass(mscClassName, IMetaStoreClient.class);

Class<? extends InvocationHandler> proxy = JavaUtils.getClass(proxyClassName, InvocationHandler.class);
try {
InvocationHandler handler = JavaUtils.newInstance(
proxy,
new Class[] { Configuration.class, Class[].class, Object[].class, ConcurrentHashMap.class, Class.class },
new Object[] { hiveConf, constructorArgTypes, constructorArgs, metaCallTimeMap, baseClass }
);
return (IMetaStoreClient) Proxy.newProxyInstance(proxy.getClassLoader(), baseClass.getInterfaces(), handler);
} catch (Throwable t) {
Throwable rootCause = ExceptionUtils.getRootCause(t);
if (rootCause instanceof Exception) {
throw ExceptionHandler.newMetaException((Exception) rootCause);
}
throw t;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ public String toString() {
ConfVars.KERBEROS_PRINCIPAL,
ConfVars.USE_THRIFT_SASL,
ConfVars.METASTORE_CLIENT_AUTH_MODE,
ConfVars.METASTORE_CLIENT_CLASS,
ConfVars.METASTORE_CLIENT_PROXY_CLASS,
ConfVars.METASTORE_CLIENT_PLAIN_USERNAME,
ConfVars.CACHE_PINOBJTYPES,
ConfVars.CONNECTION_POOLING_TYPE,
Expand Down Expand Up @@ -1660,6 +1662,14 @@ public enum ConfVars {
" and password. Any other value is ignored right now but may be used later."
+ "If JWT- Supported only in HTTP transport mode. If set, HMS Client will pick the value of JWT from "
+ "environment variable HMS_JWT and set it in Authorization header in http request"),
METASTORE_CLIENT_CLASS("metastore.client.class",
"hive.metastore.client.class",
"org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient",
"The name of the class name implementing the IMetaStoreClient interface."),
METASTORE_CLIENT_PROXY_CLASS("metastore.client.proxy.class",
"hive.metastore.client.proxy.class",
"org.apache.hadoop.hive.metastore.RetryingMetaStoreClient",
"The name of the class of the dynamic proxy to wrap an IMetaStoreClient."),
METASTORE_CLIENT_ADDITIONAL_HEADERS("metastore.client.http.additional.headers",
"hive.metastore.client.http.additional.headers", "",
"Comma separated list of headers which are passed to the metastore service in the http headers"),
Expand Down

0 comments on commit d185af2

Please sign in to comment.