From d9c40fc24236058435a5e4f59f2be0a0646a1456 Mon Sep 17 00:00:00 2001 From: levineliu Date: Tue, 19 Apr 2022 11:22:00 +0800 Subject: [PATCH 1/9] YARN-11112 Avoid renewing delegation token when app is first submitted to RM --- .../security/DelegationTokenRenewer.java | 21 +++++++++++++++---- .../security/TestDelegationTokenRenewer.java | 17 +++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index d0e0bf61748c4..61ae254f04c9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -46,6 +46,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.hadoop.security.token.TokenIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -522,10 +523,22 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) } else { tokenConf = getConfig(); } - dttr = new DelegationTokenToRenew(Arrays.asList(applicationId), token, - tokenConf, now, shouldCancelAtEnd, evt.getUser()); + // decode identify to get maxDate. + TokenIdentifier tokenIdentifier = token.decodeIdentifier(); + long expirationDate = now; + if (tokenIdentifier instanceof AbstractDelegationTokenIdentifier) { + // cast to abstract + AbstractDelegationTokenIdentifier tmpIdentifier = (AbstractDelegationTokenIdentifier) tokenIdentifier; + expirationDate = tmpIdentifier.getMaxDate(); + } + dttr = new DelegationTokenToRenew(Collections.singletonList(applicationId), token, + tokenConf, expirationDate, shouldCancelAtEnd, evt.getUser()); + try { - renewToken(dttr); + // if expire date is not greater than now, renew token. + if (expirationDate <= now) { + renewToken(dttr); + } } catch (IOException ioe) { if (ioe instanceof SecretManager.InvalidToken && dttr.maxDate < Time.now() @@ -535,7 +548,7 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) + " on recovery as it expired, requesting new hdfs token for " + applicationId + ", user=" + evt.getUser(), ioe); requestNewHdfsDelegationTokenAsProxyUser( - Arrays.asList(applicationId), evt.getUser(), + Collections.singletonList(applicationId), evt.getUser(), evt.shouldCancelAtEnd()); continue; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 2856c271f39c4..d44efd2d78aa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -50,6 +50,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -450,8 +452,15 @@ public void testDTRenewal () throws Exception { token2 = dfs.getDelegationToken("user2"); token3 = dfs.getDelegationToken("user3"); + // make token3 expire time short. + AbstractDelegationTokenIdentifier identifier = token3.decodeIdentifier(); + identifier.setMaxDate(System.currentTimeMillis()); + token3.setID(identifier.getBytes()); + //to cause this one to be set for renew in 2 secs - Renewer.tokenToRenewIn2Sec = token1; + AbstractDelegationTokenIdentifier identifier1 = token1.decodeIdentifier(); + identifier1.setMaxDate(System.currentTimeMillis() + 2 * 1000); + token1.setID(identifier1.getBytes()); LOG.info("token="+token1+" should be renewed for 2 secs"); // three distinct Namenodes @@ -473,8 +482,8 @@ public void testDTRenewal () throws Exception { new Configuration()); waitForEventsToGetProcessed(delegationTokenRenewer); - // first 3 initial renewals + 1 real - int numberOfExpectedRenewals = 3+1; + // token1 and token3 should be renewed. + int numberOfExpectedRenewals = 2; int attempts = 10; while(attempts-- > 0) { @@ -489,7 +498,7 @@ public void testDTRenewal () throws Exception { LOG.info("dfs=" + dfs.hashCode() + ";Counter = " + Renewer.counter + ";t="+ Renewer.lastRenewed); - assertEquals("renew wasn't called as many times as expected(4):", + assertEquals("renew wasn't called as many times as expected(1):", numberOfExpectedRenewals, Renewer.counter); assertEquals("most recently renewed token mismatch", Renewer.lastRenewed, token1); From d753feffc5e0c65dd521c69987547c43b960228a Mon Sep 17 00:00:00 2001 From: levineliu Date: Tue, 19 Apr 2022 15:04:49 +0800 Subject: [PATCH 2/9] fix checkstyle --- .../resourcemanager/security/DelegationTokenRenewer.java | 3 ++- .../resourcemanager/security/TestDelegationTokenRenewer.java | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index 61ae254f04c9b..6d788c75e1434 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -528,7 +528,8 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) long expirationDate = now; if (tokenIdentifier instanceof AbstractDelegationTokenIdentifier) { // cast to abstract - AbstractDelegationTokenIdentifier tmpIdentifier = (AbstractDelegationTokenIdentifier) tokenIdentifier; + AbstractDelegationTokenIdentifier tmpIdentifier = + (AbstractDelegationTokenIdentifier) tokenIdentifier; expirationDate = tmpIdentifier.getMaxDate(); } dttr = new DelegationTokenToRenew(Collections.singletonList(applicationId), token, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index d44efd2d78aa8..7e8b7a38e495c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -50,7 +50,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 610ac82003b3c0330664a24ebf20c212b6155fc0 Mon Sep 17 00:00:00 2001 From: levineliu Date: Thu, 21 Apr 2022 18:09:36 +0800 Subject: [PATCH 3/9] reuse skipTokenRenewal --- .../security/DelegationTokenRenewer.java | 32 ++++++++----------- .../security/TestDelegationTokenRenewer.java | 9 ++++-- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index 6d788c75e1434..126b3a29466ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -43,14 +43,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.hadoop.security.token.TokenIdentifier; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.DataOutputBuffer; @@ -59,8 +57,10 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -73,9 +73,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Service to renew application delegation tokens. */ @@ -492,12 +492,13 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) Set tokenList = new HashSet(); boolean hasHdfsToken = false; for (Token token : tokens) { + AtomicLong tokenExpiredTime = new AtomicLong(now); if (token.isManaged()) { if (token.getKind().equals(HDFS_DELEGATION_KIND)) { LOG.info(applicationId + " found existing hdfs token " + token); hasHdfsToken = true; } - if (skipTokenRenewal(token)) { + if (skipTokenRenewal(token, tokenExpiredTime)) { continue; } @@ -523,21 +524,12 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) } else { tokenConf = getConfig(); } - // decode identify to get maxDate. - TokenIdentifier tokenIdentifier = token.decodeIdentifier(); - long expirationDate = now; - if (tokenIdentifier instanceof AbstractDelegationTokenIdentifier) { - // cast to abstract - AbstractDelegationTokenIdentifier tmpIdentifier = - (AbstractDelegationTokenIdentifier) tokenIdentifier; - expirationDate = tmpIdentifier.getMaxDate(); - } dttr = new DelegationTokenToRenew(Collections.singletonList(applicationId), token, - tokenConf, expirationDate, shouldCancelAtEnd, evt.getUser()); + tokenConf, tokenExpiredTime.get(), shouldCancelAtEnd, evt.getUser()); try { // if expire date is not greater than now, renew token. - if (expirationDate <= now) { + if (tokenExpiredTime.get() <= now) { renewToken(dttr); } } catch (IOException ioe) { @@ -631,8 +623,9 @@ public boolean cancel() { * Skip renewing token if the renewer of the token is set to "" * Caller is expected to have examined that token.isManaged() returns * true before calling this method. + * if renewer is not empty, get the max expired time from token identifier. */ - private boolean skipTokenRenewal(Token token) + private boolean skipTokenRenewal(Token token, AtomicLong expiredTime) throws IOException { @SuppressWarnings("unchecked") @@ -641,6 +634,7 @@ private boolean skipTokenRenewal(Token token) if (identifier == null) { return false; } + expiredTime.set(identifier.getMaxDate()); Text renewer = identifier.getRenewer(); return (renewer != null && renewer.toString().equals("")); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 7e8b7a38e495c..e1c8640bf98d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -184,7 +184,7 @@ public long renew(Token t, Configuration conf) throws IOException { } @Override - public void cancel(Token t, Configuration conf) { + public void cancel(Token t, Configuration conf) throws IOException { cancelled = true; if (t instanceof MyToken) { MyToken token = (MyToken) t; @@ -308,7 +308,12 @@ public MyToken(DelegationTokenIdentifier dtId1, public boolean isCanceled() {return status.equals(CANCELED);} - public void cancelToken() {this.status=CANCELED;} + public void cancelToken() throws IOException { + this.status=CANCELED; + DelegationTokenIdentifier tokenIdentifier = this.decodeIdentifier(); + tokenIdentifier.setMaxDate(0); + this.setID(tokenIdentifier.getBytes()); + } @Override public long renew(Configuration conf) throws IOException, From c22cbe8b1db63a94cf3c51a63379b55bcb0ecccf Mon Sep 17 00:00:00 2001 From: levineliu Date: Thu, 21 Apr 2022 18:12:12 +0800 Subject: [PATCH 4/9] update test case --- .../resourcemanager/security/TestDelegationTokenRenewer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index e1c8640bf98d4..d51dde374936d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -502,7 +502,7 @@ public void testDTRenewal () throws Exception { LOG.info("dfs=" + dfs.hashCode() + ";Counter = " + Renewer.counter + ";t="+ Renewer.lastRenewed); - assertEquals("renew wasn't called as many times as expected(1):", + assertEquals("renew wasn't called as many times as expected(2):", numberOfExpectedRenewals, Renewer.counter); assertEquals("most recently renewed token mismatch", Renewer.lastRenewed, token1); From ece37d683b383236d8147e673b63cddf72f903c7 Mon Sep 17 00:00:00 2001 From: levineliu Date: Mon, 25 Apr 2022 12:13:04 +0800 Subject: [PATCH 5/9] change skipTokenRenewal name --- .../security/DelegationTokenRenewer.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index 126b3a29466ca..ef8b1c33c3229 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -57,7 +57,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -498,7 +497,7 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) LOG.info(applicationId + " found existing hdfs token " + token); hasHdfsToken = true; } - if (skipTokenRenewal(token, tokenExpiredTime)) { + if (skipTokenRenewalAndUpdateExpiredTime(token, tokenExpiredTime)) { continue; } @@ -524,7 +523,7 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) } else { tokenConf = getConfig(); } - dttr = new DelegationTokenToRenew(Collections.singletonList(applicationId), token, + dttr = new DelegationTokenToRenew(Arrays.asList(applicationId), token, tokenConf, tokenExpiredTime.get(), shouldCancelAtEnd, evt.getUser()); try { @@ -541,7 +540,7 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) + " on recovery as it expired, requesting new hdfs token for " + applicationId + ", user=" + evt.getUser(), ioe); requestNewHdfsDelegationTokenAsProxyUser( - Collections.singletonList(applicationId), evt.getUser(), + Arrays.asList(applicationId), evt.getUser(), evt.shouldCancelAtEnd()); continue; } @@ -623,9 +622,9 @@ public boolean cancel() { * Skip renewing token if the renewer of the token is set to "" * Caller is expected to have examined that token.isManaged() returns * true before calling this method. - * if renewer is not empty, get the max expired time from token identifier. + * if identifier is not null, get the max expired time from token identifier. */ - private boolean skipTokenRenewal(Token token, AtomicLong expiredTime) + private boolean skipTokenRenewalAndUpdateExpiredTime(Token token, AtomicLong expiredTime) throws IOException { @SuppressWarnings("unchecked") From 803990112d1a4a645c7e655b2f53f6079fcce3fd Mon Sep 17 00:00:00 2001 From: levineliu Date: Thu, 19 May 2022 11:40:01 +0800 Subject: [PATCH 6/9] fix imports and add clock skew --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++++ .../security/DelegationTokenRenewer.java | 13 +++++++++---- .../security/TestDelegationTokenRenewer.java | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d42562cf6140a..165bb30c8cd2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -814,6 +814,10 @@ public static boolean isAclEnabled(Configuration conf) { RM_PREFIX + "delegation-token-renewer.thread-retry-max-attempts"; public static final int DEFAULT_RM_DT_RENEWER_THREAD_RETRY_MAX_ATTEMPTS = 10; + public static final String RM_DT_RENEWER_THREAD_CLOCK_SKEW_TIME = + RM_PREFIX + "delegation-token-renewer.thread-clock-skew-time"; + public static final long DEFAULT_RM_DT_RENEWER_THREAD_CLOCK_SKEW_TIME = + TimeUnit.MINUTES.toMillis(10); // 10 minutes public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled"; public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index ef8b1c33c3229..f4a530281b707 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -46,6 +46,9 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.VisibleForTesting; @@ -72,8 +75,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Service to renew application delegation tokens. @@ -123,6 +124,7 @@ public class DelegationTokenRenewer extends AbstractService { private long tokenRenewerThreadTimeout; private long tokenRenewerThreadRetryInterval; private int tokenRenewerThreadRetryMaxAttempts; + private long clockSkewExtraTime; private final Map> futures = new ConcurrentHashMap<>(); private boolean delegationTokenRenewerPoolTrackerFlag = true; @@ -165,6 +167,8 @@ protected void serviceInit(Configuration conf) throws Exception { tokenRenewerThreadRetryMaxAttempts = conf.getInt(YarnConfiguration.RM_DT_RENEWER_THREAD_RETRY_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_DT_RENEWER_THREAD_RETRY_MAX_ATTEMPTS); + clockSkewExtraTime = conf.getLong(YarnConfiguration.RM_DT_RENEWER_THREAD_CLOCK_SKEW_TIME, + YarnConfiguration.DEFAULT_RM_DT_RENEWER_THREAD_CLOCK_SKEW_TIME); setLocalSecretManagerAndServiceAddr(); renewerService = createNewThreadPoolService(conf); pendingEventQueue = new LinkedBlockingQueue(); @@ -528,7 +532,8 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) try { // if expire date is not greater than now, renew token. - if (tokenExpiredTime.get() <= now) { + // add extra time in case of clock skew + if (tokenExpiredTime.get() <= now + clockSkewExtraTime) { renewToken(dttr); } } catch (IOException ioe) { @@ -540,7 +545,7 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) + " on recovery as it expired, requesting new hdfs token for " + applicationId + ", user=" + evt.getUser(), ioe); requestNewHdfsDelegationTokenAsProxyUser( - Arrays.asList(applicationId), evt.getUser(), + Arrays.asList(applicationId), evt.getUser(), evt.shouldCancelAtEnd()); continue; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index d51dde374936d..ca47a80f7e658 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -50,7 +50,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -69,6 +68,7 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenRenewer; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; @@ -463,7 +463,7 @@ public void testDTRenewal () throws Exception { //to cause this one to be set for renew in 2 secs AbstractDelegationTokenIdentifier identifier1 = token1.decodeIdentifier(); - identifier1.setMaxDate(System.currentTimeMillis() + 2 * 1000); + identifier1.setMaxDate(System.currentTimeMillis() + 2_000); token1.setID(identifier1.getBytes()); LOG.info("token="+token1+" should be renewed for 2 secs"); From edc1db86c7943e34773de58b97bbb9c4c7ab504d Mon Sep 17 00:00:00 2001 From: levineliu Date: Thu, 19 May 2022 11:42:57 +0800 Subject: [PATCH 7/9] fix import --- .../resourcemanager/security/DelegationTokenRenewer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index f4a530281b707..9e8a790f4194e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -51,7 +51,6 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.DataOutputBuffer; @@ -62,7 +61,6 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -76,6 +74,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder /** * Service to renew application delegation tokens. */ From 2687aff21abf6fbc4a3853d5fb4d8867a63ed759 Mon Sep 17 00:00:00 2001 From: levineliu Date: Thu, 19 May 2022 11:44:49 +0800 Subject: [PATCH 8/9] fix imports --- .../server/resourcemanager/security/DelegationTokenRenewer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index 9e8a790f4194e..fc5e406948375 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Service to renew application delegation tokens. */ From fba0d3bbb406b28bfac2812f678d8567d92bb0da Mon Sep 17 00:00:00 2001 From: levineliu Date: Thu, 19 May 2022 14:22:57 +0800 Subject: [PATCH 9/9] fix test case --- .../src/main/resources/yarn-default.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 407ef74d3d062..28558eef4b833 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1093,6 +1093,14 @@ 10 + + + Extra skew time to make sure token expire time is not affected by local machine clock + + yarn.resourcemanager.delegation-token-renewer.thread-clock-skew-time + 10 + + Time interval between each RM DelegationTokenRenewer thread retry attempt