Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
FALCON-1941 HiveDR fails with NN-HA enabled on both the source and ta…
…rget clusters

Fix for HiveDR to work NN-HA enabled clusters

Author: Venkat Ranganathan <venkat@hortonworks.com>

Reviewers: "Ying Zheng <yzheng@hortonworks.com>, Balu Vellanki <balu@apache.org>"

Closes #133 from vrangan/FALCON-1941
  • Loading branch information
Venkat Ranganathan authored and bvellanki committed May 10, 2016
1 parent 1b8a933 commit 8bd7a5c87f9c80119471b1b29a72067e6650c20e
Showing 2 changed files with 31 additions and 7 deletions.
@@ -40,8 +40,8 @@ public final class FileUtils {

private FileUtils() {}

public static Configuration getConfiguration(final String writeEP, final String nnKerberosPrincipal) {
Configuration conf = new Configuration();
public static Configuration getConfiguration(final String writeEP, final String nnKerberosPrincipal) throws IOException {
Configuration conf = HiveDRUtils.getDefaultConf();
conf.set("fs.defaultFS", writeEP);
if (StringUtils.isNotEmpty(nnKerberosPrincipal)) {
conf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal);
@@ -25,6 +25,8 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Shell;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
@@ -56,6 +58,8 @@ public enum ExecutionStage {

public static final String SEPARATOR = File.separator;

private static final Logger LOG = LoggerFactory.getLogger(HiveDRUtils.class);

private HiveDRUtils() {}

public static ReplicationType getReplicationType(List<String> sourceTables) {
@@ -65,12 +69,32 @@ public static ReplicationType getReplicationType(List<String> sourceTables) {

public static Configuration getDefaultConf() throws IOException {
Configuration conf = new Configuration();
conf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml")));
String delegationToken = getFilePathFromEnv("HADOOP_TOKEN_FILE_LOCATION");
if (delegationToken != null) {
conf.set("mapreduce.job.credentials.binary", delegationToken);
conf.set("tez.credentials.path", delegationToken);
Path confPath = new Path("file:///", System.getProperty("oozie.action.conf.xml"));

final boolean actionConfExists = confPath.getFileSystem(conf).exists(confPath);
LOG.info("Oozie Action conf {} found ? {}", confPath, actionConfExists);
if (actionConfExists) {
LOG.info("Oozie Action conf found, adding path={}, conf={}", confPath, conf.toString());
conf.addResource(confPath);
}

String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
if (StringUtils.isNotBlank(tokenFile)) {
if (Shell.WINDOWS) {
if (tokenFile.charAt(0) == '"') {
tokenFile = tokenFile.substring(1);
}
if (tokenFile.charAt(tokenFile.length() - 1) == '"') {
tokenFile = tokenFile.substring(0, tokenFile.length() - 1);
}
}

conf.set("mapreduce.job.credentials.binary", tokenFile);
System.setProperty("mapreduce.job.credentials.binary", tokenFile);
conf.set("tez.credentials.path", tokenFile);
System.setProperty("tez.credentials.path", tokenFile);
}

return conf;
}

0 comments on commit 8bd7a5c

Please sign in to comment.