svn merge -c 1335585 FIXES: MAPREDUCE-3850. Avoid redundant calls for…

… tokens in TokenCache (Daryn Sharp via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1335587 13f79535-47bb-0310-9956-ffa450edef68
apache · May 8, 2012 · 2bb2462 · 2bb2462
1 parent a431fdb
commit 2bb2462
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 1 deletion.
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
@@ -216,6 +216,9 @@ Release 0.23.3 - UNRELEASED
 
   OPTIMIZATIONS
 
+    MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
+    Sharp via bobby)
+
   BUG FIXES
 
     MAPREDUCE-4092.  commitJob Exception does not fail job (Jon Eagles via

diff --git a/...-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/...-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
@@ -19,7 +19,9 @@
 package org.apache.hadoop.mapreduce.security;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -92,8 +94,11 @@ public static void cleanUpTokenReferral(Configuration conf) {
 
   static void obtainTokensForNamenodesInternal(Credentials credentials,
       Path[] ps, Configuration conf) throws IOException {
+    Set<FileSystem> fsSet = new HashSet<FileSystem>();
     for(Path p: ps) {
-      FileSystem fs = FileSystem.get(p.toUri(), conf);
+      fsSet.add(p.getFileSystem(conf));
+    }
+    for (FileSystem fs : fsSet) {
       obtainTokensForNamenodesInternal(fs, credentials, conf);
     }
   }

diff --git a/...reduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/...reduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
@@ -251,6 +251,26 @@ public Token<?> answer(InvocationOnMock invocation) throws Throwable {
     return mockFs;
   }
 
+  @Test
+  public void testSingleTokenFetch() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
+    String renewer = Master.getMasterPrincipal(conf);
+    Credentials credentials = new Credentials();
+
+    FileSystem mockFs = mock(FileSystem.class);
+    when(mockFs.getCanonicalServiceName()).thenReturn("host:0");
+    when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0"));
+
+    Path mockPath = mock(Path.class);
+    when(mockPath.getFileSystem(conf)).thenReturn(mockFs);
+
+    Path[] paths = new Path[]{ mockPath, mockPath };
+    when(mockFs.getDelegationTokens("me", credentials)).thenReturn(null);
+    TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf);
+    verify(mockFs, times(1)).getDelegationTokens(renewer, credentials);
+  }
+
   @Test
   public void testCleanUpTokenReferral() throws Exception {
     Configuration conf = new Configuration();