Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixing a bug where we dont need to have hadoop conf in the classpath

  • Loading branch information...
commit 2965762defdc8268f91ca9ae5cc93bd2bffe5e42 1 parent c8d7a6c
Chinmay Soman authored
View
36 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java
@@ -21,7 +21,10 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.lang.reflect.Method;
import java.net.URI;
+import java.net.URL;
+import java.net.URLClassLoader;
import java.security.PrivilegedExceptionAction;
import java.text.NumberFormat;
import java.util.Arrays;
@@ -74,12 +77,15 @@
private static final int NUM_RETRIES = 3;
private String keytabLocation = "";
private String proxyUser = "voldemrt";
+ private VoldemortConfig voldemortConfig = null;
public HdfsFetcher(VoldemortConfig config) {
this(config.getMaxBytesPerSecond(),
config.getReportingIntervalBytes(),
config.getFetcherBufferSize());
+ this.voldemortConfig = config;
+
logger.info("Created hdfs fetcher with throttle rate " + maxBytesPerSecond
+ ", buffer size " + bufferSize + ", reporting interval bytes "
+ reportingIntervalBytes);
@@ -94,6 +100,8 @@ public HdfsFetcher(VoldemortConfig config, DynamicThrottleLimit dynThrottleLimit
config.getReadOnlyKeytabPath(),
config.getReadOnlyKerberosProxyUser());
+ this.voldemortConfig = config;
+
logger.info("Created hdfs fetcher with throttle rate " + dynThrottleLimit.getRate()
+ ", buffer size " + bufferSize + ", reporting interval bytes "
+ reportingIntervalBytes);
@@ -152,6 +160,8 @@ public File fetch(String sourceFileUrl, String destinationFile) throws IOExcepti
ConfigurableSocketFactory.class.getName());
FileSystem fs = null;
+ HdfsFetcher.addPath(this.voldemortConfig.getHadoopConfigPath());
+
/*
* Get the filesystem object in a secured (authenticated) block in
* case this server is talking to a Kerberized Hadoop cluster.
@@ -212,6 +222,9 @@ public FileSystem run() throws Exception {
} else {
return null;
}
+ } catch(Exception e) {
+ logger.error("Error while getting Hadoop filesystem : " + e);
+ return null;
} finally {
if(this.globalThrottleLimit != null) {
this.globalThrottleLimit.decrementNumJobs();
@@ -506,19 +519,35 @@ public void setAsyncOperationStatus(AsyncOperationStatus status) {
}
/*
+ * Function to add a resource to classpath at runtime In read-only case, it
+ * is used to load the Hadoop config in classpath
+ */
+ public static void addPath(String s) throws Exception {
+ File f = new File(s);
+ URL u = f.toURI().toURL();
+ URLClassLoader urlClassLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+ Class urlClass = URLClassLoader.class;
+ Method method = urlClass.getDeclaredMethod("addURL", new Class[] { URL.class });
+ method.setAccessible(true);
+ method.invoke(urlClassLoader, new Object[] { u });
+ }
+
+ /*
* Main method for testing fetching
*/
public static void main(String[] args) throws Exception {
if(args.length < 1)
Utils.croak("USAGE: java " + HdfsFetcher.class.getName()
- + " url [keytab location] [kerberos username]");
+ + " url [keytab location] [kerberos username] [hadoop-config-path]");
String url = args[0];
String keytabLocation = "";
String proxyUser = "";
- if(args.length >= 3) {
+ String hadoopPath = "";
+ if(args.length >= 4) {
keytabLocation = args[1];
proxyUser = args[2];
+ hadoopPath = args[3];
}
long maxBytesPerSec = 1024 * 1024 * 1024;
@@ -532,6 +561,9 @@ public static void main(String[] args) throws Exception {
config.setInt("io.socket.receive.buffer", 1 * 1024 * 1024 - 10000);
FileSystem fs = null;
+ // Add the Hadoop config to classpath
+ HdfsFetcher.addPath(hadoopPath);
+
/*
* Get the filesystem object in a secured (authenticated) block in case
* this server is talking to a Kerberized Hadoop cluster.
View
15 src/java/voldemort/server/VoldemortConfig.java
@@ -110,6 +110,7 @@
private int fetcherBufferSize;
private String readOnlyKeytabPath;
private String readOnlyKerberosProxyUser;
+ private String hadoopConfigPath;
private OpTimeMap testingSlowQueueingDelays;
private OpTimeMap testingSlowConcurrentDelays;
@@ -271,8 +272,12 @@ public VoldemortConfig(Props props) {
REPORTING_INTERVAL_BYTES);
this.fetcherBufferSize = (int) props.getBytes("hdfs.fetcher.buffer.size",
DEFAULT_BUFFER_SIZE);
- this.readOnlyKeytabPath = props.getString("readonly.keytab.path", "");
+ this.readOnlyKeytabPath = props.getString("readonly.keytab.path",
+ this.metadataDirectory
+ + "/voldemrt.headless.keytab");
this.readOnlyKerberosProxyUser = props.getString("readonly.kerberos.proxyuser", "voldemrt");
+ this.setHadoopConfigPath(props.getString("readonly.hadoop.config.path",
+ this.metadataDirectory + "/hadoop-conf"));
// TODO probably turn to false by default?
this.setUseMlock(props.getBoolean("readonly.mlock.index", true));
@@ -1602,6 +1607,14 @@ public void setReadOnlyKerberosProxyUser(String readOnlyKerberosProxyUser) {
this.readOnlyKerberosProxyUser = readOnlyKerberosProxyUser;
}
+ public String getHadoopConfigPath() {
+ return hadoopConfigPath;
+ }
+
+ public void setHadoopConfigPath(String hadoopConfigPath) {
+ this.hadoopConfigPath = hadoopConfigPath;
+ }
+
public int getSocketBufferSize() {
return socketBufferSize;
}
Please sign in to comment.
Something went wrong with that request. Please try again.