Permalink
Browse files

MAPREDUCE-7077. Pipe mapreduce job fails with Permission denied for j…

…obTokenPassword. (Akira Ajisaka via wangda)

Change-Id: Ie8f01425d58409fa3661f768205b7616128c8aa4
(cherry picked from commit 035e0f9)
  • Loading branch information...
wangdatan committed Apr 12, 2018
1 parent 9031a76 commit 995cba65fe29966583e36f9491d9a27b323918ae
@@ -45,7 +45,6 @@
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
@@ -104,8 +103,8 @@
// This password is used as shared secret key between this application and
// child pipes process
byte[] password = jobToken.getPassword();
String localPasswordFile = new File(conf.get(MRConfig.LOCAL_DIR))
+ Path.SEPARATOR + "jobTokenPassword";
String localPasswordFile = new File(".") + Path.SEPARATOR
+ "jobTokenPassword";
writePasswordToLocalFile(localPasswordFile, password, conf);
env.put("hadoop.pipes.shared.secret.location", localPasswordFile);
@@ -47,7 +47,6 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.IFile.Writer;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapred.Counters;
@@ -84,10 +83,10 @@
public void testRunner() throws Exception {
// clean old password files
JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
File[] psw = cleanTokenPasswordFile();
try {
RecordReader<FloatWritable, NullWritable> rReader = new ReaderPipesMapRunner();
JobConf conf = new JobConf();
conf.set(Submitter.IS_JAVA_RR, "true");
// for stdour and stderror
@@ -163,7 +162,7 @@ public void testApplication() throws Throwable {
TestTaskReporter reporter = new TestTaskReporter();
File[] psw = cleanTokenPasswordFile(conf);
File[] psw = cleanTokenPasswordFile();
try {
conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName);
@@ -248,7 +247,7 @@ public void testSubmitter() throws Exception {
JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
File[] psw = cleanTokenPasswordFile();
System.setProperty("test.build.data",
"target/tmp/build/TEST_SUBMITTER_MAPPER/data");
@@ -389,8 +388,8 @@ public void testSubmitter() throws Exception {
@Test
public void testPipesReduser() throws Exception {
File[] psw = cleanTokenPasswordFile();
JobConf conf = new JobConf();
File[] psw = cleanTokenPasswordFile(conf);
try {
Token<AMRMTokenIdentifier> token = new Token<AMRMTokenIdentifier>(
"user".getBytes(), "password".getBytes(), new Text("kind"), new Text(
@@ -507,16 +506,14 @@ public void progress() {
}
private File[] cleanTokenPasswordFile(JobConf conf) throws Exception {
private File[] cleanTokenPasswordFile() throws Exception {
File[] result = new File[2];
result[0] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
+ "jobTokenPassword");
result[0] = new File("./jobTokenPassword");
if (result[0].exists()) {
FileUtil.chmod(result[0].getAbsolutePath(), "700");
assertTrue(result[0].delete());
}
result[1] = new File(conf.get(MRConfig.LOCAL_DIR) + Path.SEPARATOR
+ ".jobTokenPassword.crc");
result[1] = new File("./.jobTokenPassword.crc");
if (result[1].exists()) {
FileUtil.chmod(result[1].getAbsolutePath(), "700");
result[1].delete();

0 comments on commit 995cba6

Please sign in to comment.