diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index cde4cf4841383..8c8681d29b5d7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -67,9 +67,9 @@ @InterfaceAudience.Public @InterfaceStability.Evolving -public abstract -class AbstractDelegationTokenSecretManager +public abstract +class AbstractDelegationTokenSecretManager extends SecretManager { private static final Logger LOG = LoggerFactory .getLogger(AbstractDelegationTokenSecretManager.class); @@ -84,11 +84,11 @@ private String formatTokenId(TokenIdent id) { return "(" + id + ")"; } - /** - * Cache of currently valid tokens, mapping from DelegationTokenIdentifier + /** + * Cache of currently valid tokens, mapping from DelegationTokenIdentifier * to DelegationTokenInformation. Protected by this object lock. */ - protected final Map currentTokens + protected final Map currentTokens = new ConcurrentHashMap<>(); /** @@ -102,13 +102,13 @@ private String formatTokenId(TokenIdent id) { * Protected by this object lock. */ protected int delegationTokenSequenceNumber = 0; - + /** * Access to allKeys is protected by this object lock */ - protected final Map allKeys + protected final Map allKeys = new ConcurrentHashMap<>(); - + /** * Access to currentId is protected by this object lock. */ @@ -116,12 +116,12 @@ private String formatTokenId(TokenIdent id) { /** * Access to currentKey is protected by this object lock */ - private DelegationKey currentKey; - - private long keyUpdateInterval; - private long tokenMaxLifetime; - private long tokenRemoverScanInterval; - private long tokenRenewInterval; + private volatile DelegationKey currentKey; + + private final long keyUpdateInterval; + private final long tokenMaxLifetime; + private final long tokenRemoverScanInterval; + private final long tokenRenewInterval; /** * Whether to store a token's tracking ID in its TokenInformation. * Can be overridden by a subclass. @@ -170,7 +170,7 @@ public void startThreads() throws IOException { tokenRemoverThread.start(); } } - + /** * Reset all data structures and mutable state. */ @@ -190,14 +190,14 @@ public long getCurrentTokensSize() { return currentTokens.size(); } - /** - * Add a previously used master key to cache (when NN restarts), + /** + * Add a previously used master key to cache (when NN restarts), * should be called before activate(). * * @param key delegation key. * @throws IOException raised on errors performing I/O. */ - public synchronized void addKey(DelegationKey key) throws IOException { + protected synchronized void addKey(DelegationKey key) throws IOException { if (running) // a safety check throw new IOException("Can't add delegation key to a running SecretManager."); if (key.getKeyId() > getCurrentKeyId()) { @@ -237,7 +237,7 @@ protected void storeNewToken(TokenIdent ident, long renewDate) throws IOExceptio // RM protected void removeStoredToken(TokenIdent ident) throws IOException { - + return; } // RM protected void updateStoredToken(TokenIdent ident, long renewDate) throws IOException { @@ -310,8 +310,9 @@ protected synchronized void setDelegationTokenSeqNum(int seqNum) { * * @param keyId keyId. * @return DelegationKey. + * @throws IOException raised on errors performing I/O. */ - protected DelegationKey getDelegationKey(int keyId) { + protected DelegationKey getDelegationKey(int keyId) throws IOException { return allKeys.get(keyId); } @@ -344,8 +345,9 @@ protected void updateDelegationKey(DelegationKey key) throws IOException { * * @param ident ident. * @return DelegationTokenInformation. + * @throws IOException raised on errors performing I/O. */ - protected DelegationTokenInformation getTokenInfo(TokenIdent ident) { + protected DelegationTokenInformation getTokenInfo(TokenIdent ident) throws IOException { return currentTokens.get(ident); } @@ -378,6 +380,15 @@ protected void updateToken(TokenIdent ident, updateStoredToken(ident, tokenInfo.getRenewDate()); } + protected void removeToken(TokenIdent ident) throws IOException { + DelegationTokenInformation info = currentTokens.remove(ident); + if (info == null) { + throw new InvalidToken("Token not found " + formatTokenId(ident)); + } + removeTokenForOwnerStats(ident); + removeStoredToken(ident); + } + /** * This method is intended to be used for recovering persisted delegation * tokens. Tokens that have an unknown DelegationKey are @@ -419,12 +430,13 @@ public synchronized void addPersistedDelegationToken( } } - /** - * Update the current master key - * This is called once by startThreads before tokenRemoverThread is created, + /** + * Update the current master key + * This is called once by startThreads before tokenRemoverThread is created, * and only by tokenRemoverThread afterwards. + * @throws IOException raised on errors performing I/O. */ - private void updateCurrentKey() throws IOException { + protected void updateCurrentKey() throws IOException { LOG.info("Updating the current master key for generating delegation tokens"); /* Create a new currentKey with an estimated expiry date. */ int newCurrentId; @@ -434,16 +446,17 @@ private void updateCurrentKey() throws IOException { DelegationKey newKey = new DelegationKey(newCurrentId, System .currentTimeMillis() + keyUpdateInterval + tokenMaxLifetime, generateSecret()); - //Log must be invoked outside the lock on 'this' - logUpdateMasterKey(newKey); synchronized (this) { + storeDelegationKey(newKey); + // Set current key only if storeDelegationKey is successful (doesn't throw an exception) currentKey = newKey; - storeDelegationKey(currentKey); } + //Log must be invoked outside the lock on 'this' + logUpdateMasterKey(newKey); } - - /** - * Update the current master key for generating delegation tokens + + /** + * Update the current master key for generating delegation tokens * It should be called only by tokenRemoverThread. * @throws IOException raised on errors performing I/O. */ @@ -476,11 +489,19 @@ private synchronized void removeExpiredKeys() { } } } - + @Override protected synchronized byte[] createPassword(TokenIdent identifier) { int sequenceNum; long now = Time.now(); + + // If creating new keys repeatedly fails due to state store issues, key can be expired but not yet rolled over. + // In this case, fail new token creation + if (currentKey.getExpiryDate() < now) { + String err = "Master key is expired, so unable to generate new tokens"; + LOG.error(err); + throw new RuntimeException(err); + } sequenceNum = incrementDelegationTokenSeqNum(); identifier.setIssueDate(now); identifier.setMaxDate(now + tokenMaxLifetime); @@ -496,6 +517,9 @@ protected synchronized byte[] createPassword(TokenIdent identifier) { } catch (IOException ioe) { LOG.error("Could not store token " + formatTokenId(identifier) + "!!", ioe); + // TODO - Throw IOException & handle in all clients appropriately + // Currently wrapping this as runtime exception to preserve createPassword contracts + throw new RuntimeException(ioe); } return password; } @@ -504,8 +528,7 @@ protected synchronized byte[] createPassword(TokenIdent identifier) { /** * Find the DelegationTokenInformation for the given token id, and verify that - * if the token is expired. Note that this method should be called with - * acquiring the secret manager's monitor. + * if the token is expired. * * @param identifier identifier. * @throws InvalidToken invalid token exception. @@ -513,8 +536,15 @@ protected synchronized byte[] createPassword(TokenIdent identifier) { */ protected DelegationTokenInformation checkToken(TokenIdent identifier) throws InvalidToken { - assert Thread.holdsLock(this); - DelegationTokenInformation info = getTokenInfo(identifier); + DelegationTokenInformation info = null; + try { + info = getTokenInfo(identifier); + } catch (IOException e) { + e.printStackTrace(); + // Converting IOException to InvalidTokenException as that's the current contract with checkToken + // TODO - Change this so that clients can handle accordingly + throw new InvalidToken("Token for real user: " + identifier.getRealUser() + ", can't be found in database"); + } String err; if (info == null) { err = "Token for real user: " + identifier.getRealUser() + ", can't be found in cache"; @@ -531,7 +561,7 @@ protected DelegationTokenInformation checkToken(TokenIdent identifier) } return info; } - + @Override public synchronized byte[] retrievePassword(TokenIdent identifier) throws InvalidToken { @@ -546,7 +576,12 @@ protected String getTrackingIdIfEnabled(TokenIdent ident) { } public synchronized String getTokenTrackingId(TokenIdent identifier) { - DelegationTokenInformation info = getTokenInfo(identifier); + DelegationTokenInformation info = null; + try { + info = getTokenInfo(identifier); + } catch (IOException e) { + e.printStackTrace(); + } if (info == null) { return null; } @@ -567,7 +602,7 @@ public synchronized void verifyToken(TokenIdent identifier, byte[] password) + " is invalid, password doesn't match"); } } - + /** * Renew a delegation token. * @param token the token to renew @@ -628,7 +663,7 @@ public synchronized long renewToken(Token token, METRICS.trackUpdateToken(() -> updateToken(id, info)); return renewTime; } - + /** * Cancel a token by removing it from cache. * @@ -646,7 +681,7 @@ public synchronized TokenIdent cancelToken(Token token, id.readFields(in); LOG.info("Token cancellation requested for identifier: " + formatTokenId(id)); - + if (id.getUser() == null) { throw new InvalidToken("Token with no owner " + formatTokenId(id)); } @@ -660,17 +695,12 @@ public synchronized TokenIdent cancelToken(Token token, throw new AccessControlException(canceller + " is not authorized to cancel the token " + formatTokenId(id)); } - DelegationTokenInformation info = currentTokens.remove(id); - if (info == null) { - throw new InvalidToken("Token not found " + formatTokenId(id)); - } METRICS.trackRemoveToken(() -> { - removeTokenForOwnerStats(id); - removeStoredToken(id); + removeToken(id); }); return id; } - + /** * Convert the byte[] to a secret key * @param key the byte[] to create the secret key from @@ -683,9 +713,9 @@ public static SecretKey createSecretKey(byte[] key) { /** Class to encapsulate a token's renew date and password. */ @InterfaceStability.Evolving public static class DelegationTokenInformation implements Writable { - long renewDate; - byte[] password; - String trackingId; + private long renewDate; + private byte[] password; + private String trackingId; public DelegationTokenInformation() { this(0, null); @@ -744,7 +774,7 @@ public void readFields(DataInput in) throws IOException { trackingId = WritableUtils.readString(in); } } - + /** Remove expired delegation tokens from cache */ private void removeExpiredToken() throws IOException { long now = Time.now(); @@ -779,7 +809,7 @@ public void stopThreads() { if (LOG.isDebugEnabled()) LOG.debug("Stopping expired delegation token remover thread"); running = false; - + if (tokenRemoverThread != null) { synchronized (noInterruptsLock) { tokenRemoverThread.interrupt(); @@ -792,7 +822,7 @@ public void stopThreads() { } } } - + /** * is secretMgr running * @return true if secret mgr is running @@ -800,7 +830,7 @@ public void stopThreads() { public synchronized boolean isRunning() { return running; } - + private class ExpiredTokenRemover extends Thread { private long lastMasterKeyUpdate; private long lastTokenCacheCleanup; @@ -841,7 +871,7 @@ public void run() { /** * Decode the token identifier. The subclass can customize the way to decode * the token identifier. - * + * * @param token the token where to extract the identifier * @return the delegation token identifier * @throws IOException raised on errors performing I/O. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java index 4b6ae21d7a95b..c797438d7d8f4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/SQLDelegationTokenSecretManager.java @@ -161,7 +161,7 @@ protected void removeStoredToken(TokenIdent ident) throws IOException { * null if it doesn't exist in the database. */ @Override - protected DelegationTokenInformation getTokenInfo(TokenIdent ident) { + protected DelegationTokenInformation getTokenInfo(TokenIdent ident) throws IOException { // Look for token in local cache DelegationTokenInformation tokenInfo = super.getTokenInfo(ident); @@ -302,7 +302,7 @@ protected void removeStoredMasterKey(DelegationKey key) { * if it doesn't exist in the database. */ @Override - protected DelegationKey getDelegationKey(int keyId) { + protected DelegationKey getDelegationKey(int keyId) throws IOException { // Look for delegation key in local cache DelegationKey delegationKey = super.getDelegationKey(keyId); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index fb9a2951f598a..5bd0eab796f51 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -569,7 +569,7 @@ protected int incrementCurrentKeyId() { } @Override - protected DelegationKey getDelegationKey(int keyId) { + protected DelegationKey getDelegationKey(int keyId) throws IOException { // First check if its I already have this key DelegationKey key = allKeys.get(keyId); // Then query ZK @@ -581,6 +581,7 @@ protected DelegationKey getDelegationKey(int keyId) { } } catch (IOException e) { LOG.error("Error retrieving key [" + keyId + "] from ZK", e); + throw e; } } return key; @@ -608,7 +609,7 @@ private DelegationKey getKeyFromZK(int keyId) throws IOException { } @Override - protected DelegationTokenInformation getTokenInfo(TokenIdent ident) { + protected DelegationTokenInformation getTokenInfo(TokenIdent ident) throws IOException { // First check if I have this.. DelegationTokenInformation tokenInfo = currentTokens.get(ident); // Then query ZK @@ -621,6 +622,7 @@ protected DelegationTokenInformation getTokenInfo(TokenIdent ident) { } catch (IOException e) { LOG.error("Error retrieving tokenInfo [" + ident.getSequenceNumber() + "] from ZK", e); + throw e; } } return tokenInfo; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java index 225cc658d39ba..6785d9af5c2c0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java @@ -727,7 +727,7 @@ private void callAndValidateFailureMetrics(TestDelegationTokenSecretManager if (expectError) { LambdaTestUtils.intercept(IOException.class, callable); } else { - callable.call(); + Assert.assertThrows(Exception.class, () -> callable.call()); } assertEquals(counterBefore + 1, counter.value()); assertEquals(statBefore + 1, failureStat.getSamples()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java index e92a25ea0ed8f..d4945bc61e091 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java @@ -473,9 +473,13 @@ public void testNodesLoadedAfterRestart() throws Exception { GenericTestUtils.waitFor(new Supplier() { @Override public Boolean get() { - AbstractDelegationTokenSecretManager.DelegationTokenInformation dtinfo = - zksm.getTokenInfo(idCancelled); - return dtinfo == null; + try { + AbstractDelegationTokenSecretManager.DelegationTokenInformation dtinfo = + zksm.getTokenInfo(idCancelled); + return dtinfo == null; + } catch (IOException ex) { + return false; + } } }, 100, 5000); @@ -508,7 +512,11 @@ public Boolean get() { @Override public Boolean get() { LOG.info("Waiting for the expired token to be removed..."); - return zksm1.getTokenInfo(id1) == null; + try { + return zksm1.getTokenInfo(id1) == null; + } catch (Exception ex) { + return false; + } } }, 1000, 5000); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index 452bcf9d4ad87..15a34d7e2d540 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -114,35 +114,14 @@ RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) throws YarnException, IOException; /** - * The Router Supports incrementDelegationTokenSeqNum. + * Return a new unique integer lookup key for a new delegation token * * @return DelegationTokenSeqNum. */ int incrementDelegationTokenSeqNum(); /** - * The Router Supports getDelegationTokenSeqNum. - * - * @return DelegationTokenSeqNum. - */ - int getDelegationTokenSeqNum(); - - /** - * The Router Supports setDelegationTokenSeqNum. - * - * @param seqNum DelegationTokenSeqNum. - */ - void setDelegationTokenSeqNum(int seqNum); - - /** - * The Router Supports getCurrentKeyId. - * - * @return CurrentKeyId. - */ - int getCurrentKeyId(); - - /** - * The Router Supports incrementCurrentKeyId. + * Return a new unique integer master key id * * @return CurrentKeyId. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 273e736e88774..f655bb36d9621 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -138,10 +138,7 @@ public void init(Configuration conf) { @Override public void close() { - membership = null; - applications = null; - reservations = null; - policies = null; + init(new Configuration()); } @Override @@ -464,13 +461,13 @@ public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) RouterMasterKey masterKey = request.getRouterMasterKey(); DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey); - Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); - if (rmDTMasterKeyState.contains(delegationKey)) { + Map rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + if (rmDTMasterKeyState.containsKey(delegationKey.getKeyId())) { FederationStateStoreUtils.logAndThrowStoreException(LOG, "Error storing info for RMDTMasterKey with keyID: %s.", delegationKey.getKeyId()); } - routerRMSecretManagerState.getMasterKeyState().add(delegationKey); + routerRMSecretManagerState.getMasterKeyState().put(delegationKey.getKeyId(), delegationKey); LOG.info("Store Router-RMDT master key with key id: {}. Currently rmDTMasterKeyState size: {}", delegationKey.getKeyId(), rmDTMasterKeyState.size()); @@ -485,8 +482,8 @@ public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest requ DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey); LOG.info("Remove Router-RMDT master key with key id: {}.", delegationKey.getKeyId()); - Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); - rmDTMasterKeyState.remove(delegationKey); + Map rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + rmDTMasterKeyState.remove(delegationKey.getKeyId()); return RouterMasterKeyResponse.newInstance(masterKey); } @@ -494,12 +491,10 @@ public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest requ @Override public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) throws YarnException, IOException { - // Restore the DelegationKey from the request RouterMasterKey masterKey = request.getRouterMasterKey(); - DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey); - - Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); - if (!rmDTMasterKeyState.contains(delegationKey)) { + Map rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + DelegationKey delegationKey = rmDTMasterKeyState.get(masterKey.getKeyId()); + if (delegationKey == null) { throw new IOException("GetMasterKey with keyID: " + masterKey.getKeyId() + " does not exist."); } @@ -524,9 +519,9 @@ public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); - Map rmDTState = + Map rmDTState = routerRMSecretManagerState.getTokenState(); - rmDTState.remove(tokenIdentifier); + rmDTState.remove(tokenIdentifier.getSequenceNumber()); storeOrUpdateRouterRMDT(tokenIdentifier, storeToken, true); return RouterRMTokenResponse.newInstance(storeToken); } @@ -537,9 +532,9 @@ public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); - Map rmDTState = + Map rmDTState = routerRMSecretManagerState.getTokenState(); - rmDTState.remove(tokenIdentifier); + rmDTState.remove(tokenIdentifier.getSequenceNumber()); return RouterRMTokenResponse.newInstance(storeToken); } @@ -549,13 +544,13 @@ public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest req RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); - Map rmDTState = + Map rmDTState = routerRMSecretManagerState.getTokenState(); - if (!rmDTState.containsKey(tokenIdentifier)) { + if (!rmDTState.containsKey(tokenIdentifier.getSequenceNumber())) { LOG.info("Router RMDelegationToken: {} does not exist.", tokenIdentifier); throw new IOException("Router RMDelegationToken: " + tokenIdentifier + " does not exist."); } - RouterStoreToken resultToken = rmDTState.get(tokenIdentifier); + RouterStoreToken resultToken = rmDTState.get(tokenIdentifier.getSequenceNumber()); return RouterRMTokenResponse.newInstance(resultToken); } @@ -564,21 +559,6 @@ public int incrementDelegationTokenSeqNum() { return sequenceNum.incrementAndGet(); } - @Override - public int getDelegationTokenSeqNum() { - return sequenceNum.get(); - } - - @Override - public void setDelegationTokenSeqNum(int seqNum) { - sequenceNum.set(seqNum); - } - - @Override - public int getCurrentKeyId() { - return masterKeyId.get(); - } - @Override public int incrementCurrentKeyId() { return masterKeyId.incrementAndGet(); @@ -586,13 +566,13 @@ public int incrementCurrentKeyId() { private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier, RouterStoreToken routerStoreToken, boolean isUpdate) throws IOException { - Map rmDTState = + Map rmDTState = routerRMSecretManagerState.getTokenState(); - if (rmDTState.containsKey(rmDTIdentifier)) { + if (rmDTState.containsKey(rmDTIdentifier.getSequenceNumber())) { LOG.info("Error storing info for RMDelegationToken: {}.", rmDTIdentifier); throw new IOException("Router RMDelegationToken: " + rmDTIdentifier + "is already stored."); } - rmDTState.put(rmDTIdentifier, routerStoreToken); + rmDTState.put(rmDTIdentifier.getSequenceNumber(), routerStoreToken); if (!isUpdate) { routerRMSecretManagerState.setDtSequenceNumber(rmDTIdentifier.getSequenceNumber()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index e1fc3f2a47e4c..98792c272dd98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -1850,45 +1850,6 @@ public int incrementDelegationTokenSeqNum() { return querySequenceTable(YARN_ROUTER_SEQUENCE_NUM, true); } - /** - * Get DelegationToken SeqNum. - * - * @return delegationTokenSeqNum. - */ - @Override - public int getDelegationTokenSeqNum() { - return querySequenceTable(YARN_ROUTER_SEQUENCE_NUM, false); - } - - @Override - public void setDelegationTokenSeqNum(int seqNum) { - Connection connection = null; - try { - connection = getConnection(false); - FederationQueryRunner runner = new FederationQueryRunner(); - runner.updateSequenceTable(connection, YARN_ROUTER_SEQUENCE_NUM, seqNum); - } catch (Exception e) { - throw new RuntimeException("Could not update sequence table!!", e); - } finally { - // Return to the pool the CallableStatement - try { - FederationStateStoreUtils.returnToPool(LOG, null, connection); - } catch (YarnException e) { - LOG.error("close connection error.", e); - } - } - } - - /** - * Get Current KeyId. - * - * @return currentKeyId. - */ - @Override - public int getCurrentKeyId() { - return querySequenceTable(YARN_ROUTER_CURRENT_KEY_ID, false); - } - /** * The Router Supports incrementCurrentKeyId. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index 536faa31dca0c..5d618c6cae7c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -1553,40 +1553,6 @@ private int incrSharedCount(SharedCount sharedCount, int batchSize) } } - /** - * Get DelegationToken SeqNum. - * - * @return delegationTokenSeqNum. - */ - @Override - public int getDelegationTokenSeqNum() { - return delTokSeqCounter.getCount(); - } - - /** - * Set DelegationToken SeqNum. - * - * @param seqNum sequenceNum. - */ - @Override - public void setDelegationTokenSeqNum(int seqNum) { - try { - delTokSeqCounter.setCount(seqNum); - } catch (Exception e) { - throw new RuntimeException("Could not set shared counter !!", e); - } - } - - /** - * Get Current KeyId. - * - * @return currentKeyId. - */ - @Override - public int getCurrentKeyId() { - return keyIdSeqCounter.getCount(); - } - /** * The Router Supports incrementCurrentKeyId. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java index 62a89f419dd6a..7704946ee8249 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java @@ -28,17 +28,17 @@ public class RouterRMDTSecretManagerState { // DTIdentifier -> renewDate - private Map delegationTokenState = new HashMap<>(); + private Map delegationTokenState = new HashMap<>(); - private Set masterKeyState = new HashSet<>(); + private Map masterKeyState = new HashMap<>(); private int dtSequenceNumber = 0; - public Map getTokenState() { + public Map getTokenState() { return delegationTokenState; } - public Set getMasterKeyState() { + public Map getMasterKeyState() { return masterKeyState; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index ebad527b6d452..d4e14c21b34b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -780,17 +780,14 @@ public void removeStoredMasterKey(DelegationKey newKey) throws YarnException, IO /** * The Router Supports GetMasterKeyByDelegationKey. * - * @param newKey Key used for generating and verifying delegation tokens + * @param keyId Master Key identifer that is used to generate the delegation tokens * @throws YarnException if the call to the state store is unsuccessful * @throws IOException An IO Error occurred * @return RouterMasterKeyResponse */ - public RouterMasterKeyResponse getMasterKeyByDelegationKey(DelegationKey newKey) - throws YarnException, IOException { - LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); - ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); - RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), - keyBytes, newKey.getExpiryDate()); + public RouterMasterKeyResponse getMasterKey(int keyId) throws YarnException, IOException { + // TODO - Change interfaces with state store to get master key from key id + RouterMasterKey masterKey = RouterMasterKey.newInstance(keyId, null, 0L); RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey); return stateStore.getMasterKeyByDelegationKey(keyRequest); } @@ -907,33 +904,6 @@ public int incrementDelegationTokenSeqNum() { return stateStore.incrementDelegationTokenSeqNum(); } - /** - * Get SeqNum from stateStore. - * - * @return delegationTokenSequenceNumber. - */ - public int getDelegationTokenSeqNum() { - return stateStore.getDelegationTokenSeqNum(); - } - - /** - * Set SeqNum from stateStore. - * - * @param seqNum delegationTokenSequenceNumber. - */ - public void setDelegationTokenSeqNum(int seqNum) { - stateStore.setDelegationTokenSeqNum(seqNum); - } - - /** - * Get CurrentKeyId from stateStore. - * - * @return currentKeyId. - */ - public int getCurrentKeyId() { - return stateStore.getCurrentKeyId(); - } - /** * stateStore provides CurrentKeyId increase. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java index 5548dab1b8c06..3aa07b7a3e5c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; -import java.util.Set; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -59,10 +58,10 @@ protected void checkRouterMasterKey(DelegationKey delegationKey, memoryStateStore.getRouterRMSecretManagerState(); assertNotNull(secretManagerState); - Set delegationKeys = secretManagerState.getMasterKeyState(); + Map delegationKeys = secretManagerState.getMasterKeyState(); assertNotNull(delegationKeys); - assertTrue(delegationKeys.contains(delegationKey)); + assertTrue(delegationKeys.containsKey(delegationKey.getKeyId())); RouterMasterKey resultRouterMasterKey = RouterMasterKey.newInstance(delegationKey.getKeyId(), ByteBuffer.wrap(delegationKey.getEncodedKey()), delegationKey.getExpiryDate()); @@ -78,11 +77,11 @@ protected void checkRouterStoreToken(RMDelegationTokenIdentifier identifier, memoryStateStore.getRouterRMSecretManagerState(); assertNotNull(secretManagerState); - Map tokenStateMap = + Map tokenStateMap = secretManagerState.getTokenState(); assertNotNull(tokenStateMap); - assertTrue(tokenStateMap.containsKey(identifier)); + assertTrue(tokenStateMap.containsKey(identifier.getSequenceNumber())); YARNDelegationTokenIdentifier tokenIdentifier = token.getTokenIdentifier(); assertTrue(tokenIdentifier instanceof RMDelegationTokenIdentifier); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index 92dd426f51372..07d8fce8ffe62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.HashSet; @@ -248,8 +249,8 @@ public void testGetApplicationHomeSubClusterCache() throws YarnException { public void testStoreNewMasterKey() throws YarnException, IOException { // store delegation key; DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes()); - Set keySet = new HashSet<>(); - keySet.add(key); + Map keySet = new HashMap<>(); + keySet.put(key.getKeyId(), key); facade.storeNewMasterKey(key); MemoryFederationStateStore federationStateStore = @@ -263,8 +264,8 @@ public void testStoreNewMasterKey() throws YarnException, IOException { public void testRemoveStoredMasterKey() throws YarnException, IOException { // store delegation key; DelegationKey key = new DelegationKey(4567, 7654, "keyBytes".getBytes()); - Set keySet = new HashSet<>(); - keySet.add(key); + Map keySet = new HashMap<>(); + keySet.put(key.getKeyId(), key); facade.storeNewMasterKey(key); // check to delete delegationKey diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index 90dcadb721ec7..70c65af400bac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -438,21 +438,6 @@ public int incrementDelegationTokenSeqNum() { return stateStoreClient.incrementDelegationTokenSeqNum(); } - @Override - public int getDelegationTokenSeqNum() { - return stateStoreClient.getDelegationTokenSeqNum(); - } - - @Override - public void setDelegationTokenSeqNum(int seqNum) { - stateStoreClient.setDelegationTokenSeqNum(seqNum); - } - - @Override - public int getCurrentKeyId() { - return stateStoreClient.getCurrentKeyId(); - } - @Override public int incrementCurrentKeyId() { return stateStoreClient.incrementCurrentKeyId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/security/token/delegation/RouterDelegationTokenSupport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/security/token/delegation/RouterDelegationTokenSupport.java index d530f751cb20a..daf340a9bbaec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/security/token/delegation/RouterDelegationTokenSupport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/security/token/delegation/RouterDelegationTokenSupport.java @@ -41,9 +41,9 @@ public static String encodeDelegationTokenInformation(DelegationTokenInformation try { ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(bos); - WritableUtils.writeVInt(out, token.password.length); - out.write(token.password); - out.writeLong(token.renewDate); + WritableUtils.writeVInt(out, token.getPassword().length); + out.write(token.getPassword()); + out.writeLong(token.getRenewDate()); out.flush(); byte[] tokenInfoBytes = bos.toByteArray(); return Base64.getUrlEncoder().encodeToString(tokenInfoBytes); @@ -55,11 +55,11 @@ public static String encodeDelegationTokenInformation(DelegationTokenInformation public static DelegationTokenInformation decodeDelegationTokenInformation(byte[] tokenBytes) throws IOException { DataInputStream in = new DataInputStream(new ByteArrayInputStream(tokenBytes)); - DelegationTokenInformation token = new DelegationTokenInformation(0, null); int len = WritableUtils.readVInt(in); - token.password = new byte[len]; - in.readFully(token.password); - token.renewDate = in.readLong(); + byte[] password = new byte[len]; + in.readFully(password); + long renewDate = in.readLong(); + DelegationTokenInformation token = new DelegationTokenInformation(renewDate, password); return token; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java index 57d2aaa4bf147..867990472ec24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -17,16 +17,12 @@ */ package org.apache.hadoop.yarn.server.router.security; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.security.token.delegation.RouterDelegationTokenSupport; -import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; -import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; @@ -37,16 +33,27 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; import java.util.Base64; /** - * A Router specific delegation token secret manager. + * A Router specific delegation token secret manager and is designed to be stateless. * The secret manager is responsible for generating and accepting the password * for each token. + * + * Behavioural Differences from AbstractDelegationTokenSecretManager + * 1) Master Key - Each instance of Router will have its own master key and each instance rolls its own master key. + * Thus there is no concept of a global current key. + * The requirement to generate new master keys / delegation tokens is to generate unique INTEGER keys, + * which the state store is responsible for (Autoincrement is one of the ways to achieve this). + * This key will be regenerated on service restart and thus there is no requirement of an explicit restore mechanism. + * Current master key will be stored in memory on each instance and will be used to generate new tokens. + * Master key will be looked up from the state store for Validation / renewal, etc of tokens. + * + * 2) Token Expiry - It doesn't take care of token removal on expiry. + * Each state store can implement its own way to manage token deletion on expiry. + * + * This pretty much replaces all methods of AbstractDelegationTokenSecretManager which is designed for stateful managers + * TODO - Refactor Secret Manager interfaces to support stateful and stateless secret management */ public class RouterDelegationTokenSecretManager extends AbstractDelegationTokenSecretManager { @@ -76,12 +83,31 @@ public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, } @Override - public RMDelegationTokenIdentifier createIdentifier() { - return new RMDelegationTokenIdentifier(); + public void reset() { + // no-op } - private boolean shouldIgnoreException(Exception e) { - return !running && e.getCause() instanceof InterruptedException; + @Override + public long getCurrentTokensSize() { + throw new NotImplementedException("Get current token size is not implemented"); + } + + /** + * no-op as this method is required for stateful secret managers only + */ + @Override + protected void addKey(DelegationKey key) { + + } + + @Override + public DelegationKey[] getAllKeys() { + throw new NotImplementedException("Get all keys is not implemented"); + } + + @Override + public RMDelegationTokenIdentifier createIdentifier() { + return new RMDelegationTokenIdentifier(); } /** @@ -91,283 +117,173 @@ private boolean shouldIgnoreException(Exception e) { * @param newKey DelegationKey */ @Override - public void storeNewMasterKey(DelegationKey newKey) { + protected void storeNewMasterKey(DelegationKey newKey) throws IOException { try { federationFacade.storeNewMasterKey(newKey); - } catch (Exception e) { - if (!shouldIgnoreException(e)) { - LOG.error("Error in storing master key with KeyID: {}.", newKey.getKeyId()); - ExitUtil.terminate(1, e); - } + } catch (YarnException e) { + e.printStackTrace(); + throw new IOException(e); // Wrap YarnException as an IOException to adhere to storeNewMasterKey contract } } /** - * The Router Supports Remove the master key. - * During this Process, Facade will call the specific StateStore to remove the MasterKey. - * - * @param delegationKey DelegationKey + * no-op as expiry of stored keys is upto the state store in a stateless secret manager */ @Override public void removeStoredMasterKey(DelegationKey delegationKey) { - try { - federationFacade.removeStoredMasterKey(delegationKey); - } catch (Exception e) { - if (!shouldIgnoreException(e)) { - LOG.error("Error in removing master key with KeyID: {}.", delegationKey.getKeyId()); - ExitUtil.terminate(1, e); - } - } + } /** - * The Router Supports Store new Token. - * - * @param identifier RMDelegationToken - * @param renewDate renewDate - * @throws IOException IO exception occurred. + * no-op as we are storing entire token with info in storeToken() */ @Override - public void storeNewToken(RMDelegationTokenIdentifier identifier, - long renewDate) throws IOException { - try { - federationFacade.storeNewToken(identifier, renewDate); - } catch (Exception e) { - if (!shouldIgnoreException(e)) { - LOG.error("Error in storing RMDelegationToken with sequence number: {}.", - identifier.getSequenceNumber()); - ExitUtil.terminate(1, e); - } - } - } + public void storeNewToken(RMDelegationTokenIdentifier identifier, long renewDate) { - /** - * The Router Supports Store new Token. - * - * @param identifier RMDelegationToken. - * @param tokenInfo DelegationTokenInformation. - */ - public void storeNewToken(RMDelegationTokenIdentifier identifier, - DelegationTokenInformation tokenInfo) { - try { - String token = - RouterDelegationTokenSupport.encodeDelegationTokenInformation(tokenInfo); - long renewDate = tokenInfo.getRenewDate(); - - federationFacade.storeNewToken(identifier, renewDate, token); - } catch (Exception e) { - if (!shouldIgnoreException(e)) { - LOG.error("Error in storing RMDelegationToken with sequence number: {}.", - identifier.getSequenceNumber()); - ExitUtil.terminate(1, e); - } - } } /** - * The Router Supports Update Token. - * - * @param id RMDelegationToken - * @param renewDate renewDate - * @throws IOException IO exception occurred + * no-op as removal of tokens is handled in removeToken() */ @Override - public void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) throws IOException { - try { - federationFacade.updateStoredToken(id, renewDate); - } catch (Exception e) { - if (!shouldIgnoreException(e)) { - LOG.error("Error in updating persisted RMDelegationToken with sequence number: {}.", - id.getSequenceNumber()); - ExitUtil.terminate(1, e); - } - } - } + public void removeStoredToken(RMDelegationTokenIdentifier identifier) { - /** - * The Router Supports Update Token. - * - * @param identifier RMDelegationToken. - * @param tokenInfo DelegationTokenInformation. - */ - public void updateStoredToken(RMDelegationTokenIdentifier identifier, - DelegationTokenInformation tokenInfo) { - try { - long renewDate = tokenInfo.getRenewDate(); - String token = RouterDelegationTokenSupport.encodeDelegationTokenInformation(tokenInfo); - federationFacade.updateStoredToken(identifier, renewDate, token); - } catch (Exception e) { - if (!shouldIgnoreException(e)) { - LOG.error("Error in updating persisted RMDelegationToken with sequence number: {}.", - identifier.getSequenceNumber()); - ExitUtil.terminate(1, e); - } - } } /** - * The Router Supports Remove Token. - * - * @param identifier Delegation Token - * @throws IOException IO exception occurred. + * no-op as we are storing entire token with info in updateToken() */ @Override - public void removeStoredToken(RMDelegationTokenIdentifier identifier) throws IOException { - try { - federationFacade.removeStoredToken(identifier); - } catch (Exception e) { - if (!shouldIgnoreException(e)) { - LOG.error("Error in removing RMDelegationToken with sequence number: {}", - identifier.getSequenceNumber()); - ExitUtil.terminate(1, e); - } - } + public void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) { + } - /** - * The Router supports obtaining the DelegationKey stored in the Router StateStote - * according to the DelegationKey. - * - * @param key Param DelegationKey - * @return Delegation Token - * @throws YarnException An internal conversion error occurred when getting the Token - * @throws IOException IO exception occurred - */ - public DelegationKey getMasterKeyByDelegationKey(DelegationKey key) - throws YarnException, IOException { + @Override + public DelegationKey getDelegationKey(int keyId) throws IOException { try { - RouterMasterKeyResponse response = federationFacade.getMasterKeyByDelegationKey(key); + RouterMasterKeyResponse response = federationFacade.getMasterKey(keyId); RouterMasterKey masterKey = response.getRouterMasterKey(); ByteBuffer keyByteBuf = masterKey.getKeyBytes(); byte[] keyBytes = new byte[keyByteBuf.remaining()]; keyByteBuf.get(keyBytes); - DelegationKey delegationKey = - new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); - return delegationKey; - } catch (IOException ex) { - throw new IOException(ex); + return new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); } catch (YarnException ex) { - throw new YarnException(ex); - } - } - - /** - * Get RMDelegationTokenIdentifier according to RouterStoreToken. - * - * @param identifier RMDelegationTokenIdentifier - * @return RMDelegationTokenIdentifier - * @throws YarnException An internal conversion error occurred when getting the Token - * @throws IOException IO exception occurred - */ - public RMDelegationTokenIdentifier getTokenByRouterStoreToken( - RMDelegationTokenIdentifier identifier) throws YarnException, IOException { - try { - RouterRMTokenResponse response = federationFacade.getTokenByRouterStoreToken(identifier); - YARNDelegationTokenIdentifier responseIdentifier = - response.getRouterStoreToken().getTokenIdentifier(); - return (RMDelegationTokenIdentifier) responseIdentifier; - } catch (Exception ex) { - throw new YarnException(ex); - } - } - - public void setFederationFacade(FederationStateStoreFacade federationFacade) { - this.federationFacade = federationFacade; - } - - @Public - @VisibleForTesting - public int getLatestDTSequenceNumber() { - return delegationTokenSequenceNumber; - } - - @Public - @VisibleForTesting - public synchronized Set getAllMasterKeys() { - return new HashSet<>(allKeys.values()); - } - - @Public - @VisibleForTesting - public synchronized Map getAllTokens() { - Map allTokens = new HashMap<>(); - for (Map.Entry entry : currentTokens.entrySet()) { - RMDelegationTokenIdentifier keyIdentifier = entry.getKey(); - DelegationTokenInformation tokenInformation = entry.getValue(); - allTokens.put(keyIdentifier, tokenInformation.getRenewDate()); + ex.printStackTrace(); + throw new IOException(ex); } - return allTokens; } - public long getRenewDate(RMDelegationTokenIdentifier ident) - throws InvalidToken { - DelegationTokenInformation info = currentTokens.get(ident); - if (info == null) { - throw new InvalidToken("token (" + ident.toString() - + ") can't be found in cache"); - } + public long getRenewDate(RMDelegationTokenIdentifier ident) throws IOException { + DelegationTokenInformation info = getTokenInfo(ident); return info.getRenewDate(); } @Override - protected synchronized int incrementDelegationTokenSeqNum() { + protected int incrementDelegationTokenSeqNum() { return federationFacade.incrementDelegationTokenSeqNum(); } @Override protected void storeToken(RMDelegationTokenIdentifier rmDelegationTokenIdentifier, DelegationTokenInformation tokenInfo) throws IOException { - this.currentTokens.put(rmDelegationTokenIdentifier, tokenInfo); - this.addTokenForOwnerStats(rmDelegationTokenIdentifier); - storeNewToken(rmDelegationTokenIdentifier, tokenInfo); + try { + storeNewToken(rmDelegationTokenIdentifier, tokenInfo); + } catch (YarnException e) { + e.printStackTrace(); + throw new IOException(e); // Wrap YarnException as an IOException to adhere to storeToken contract + } } @Override protected void updateToken(RMDelegationTokenIdentifier rmDelegationTokenIdentifier, DelegationTokenInformation tokenInfo) throws IOException { - this.currentTokens.put(rmDelegationTokenIdentifier, tokenInfo); - updateStoredToken(rmDelegationTokenIdentifier, tokenInfo); + try { + updateStoredToken(rmDelegationTokenIdentifier, tokenInfo); + } catch (YarnException e) { + e.printStackTrace(); + throw new IOException(e); // Wrap YarnException as an IOException to adhere to updateToken contract + } } @Override - protected DelegationTokenInformation getTokenInfo( - RMDelegationTokenIdentifier ident) { - // First check if I have this.. - DelegationTokenInformation tokenInfo = currentTokens.get(ident); - if (tokenInfo == null) { + protected void removeToken(RMDelegationTokenIdentifier identifier) throws IOException { + try { + federationFacade.removeStoredToken(identifier); + } catch (YarnException e) { + e.printStackTrace(); + throw new IOException(e); // Wrap YarnException as an IOException to adhere to removeToken contract + } + } + + @Override + protected DelegationTokenInformation getTokenInfo(RMDelegationTokenIdentifier ident) throws IOException { try { RouterRMTokenResponse response = federationFacade.getTokenByRouterStoreToken(ident); RouterStoreToken routerStoreToken = response.getRouterStoreToken(); String tokenStr = routerStoreToken.getTokenInfo(); byte[] tokenBytes = Base64.getUrlDecoder().decode(tokenStr); - tokenInfo = RouterDelegationTokenSupport.decodeDelegationTokenInformation(tokenBytes); - } catch (Exception e) { - LOG.error("Error retrieving tokenInfo [" + ident.getSequenceNumber() - + "] from StateStore.", e); - throw new YarnRuntimeException(e); + return RouterDelegationTokenSupport.decodeDelegationTokenInformation(tokenBytes); + } catch (YarnException ex) { + ex.printStackTrace(); + throw new IOException(ex); } - } - return tokenInfo; } @Override - protected synchronized int getDelegationTokenSeqNum() { - return federationFacade.getDelegationTokenSeqNum(); + protected void rollMasterKey() throws IOException { + updateCurrentKey(); } @Override - protected synchronized void setDelegationTokenSeqNum(int seqNum) { - federationFacade.setDelegationTokenSeqNum(seqNum); + public void addPersistedDelegationToken(RMDelegationTokenIdentifier identifier, long renewDate) { + throw new NotImplementedException("Recovery of tokens is not a valid use case for stateless secret managers"); } @Override - protected synchronized int getCurrentKeyId() { - return federationFacade.getCurrentKeyId(); + protected int getDelegationTokenSeqNum() { + throw new NotImplementedException("Get sequence number is not a valid use case for stateless secret managers"); } @Override - protected synchronized int incrementCurrentKeyId() { + protected void setDelegationTokenSeqNum(int seqNum) { + throw new NotImplementedException("Set sequence number is not a valid use case for stateless secret managers"); + } + + @Override + protected int getCurrentKeyId() { + throw new NotImplementedException("Get current key id is not a valid use case for stateless secret managers"); + } + + @Override + protected int incrementCurrentKeyId() { return federationFacade.incrementCurrentKeyId(); } + + @Override + protected void setCurrentKeyId(int keyId) { + throw new NotImplementedException("Set current key id is not a valid use case for stateless secret managers"); + } + + @Override + protected void storeDelegationKey(DelegationKey key) throws IOException { + storeNewMasterKey(key); + } + + @Override + protected void updateDelegationKey(DelegationKey key) { + throw new NotImplementedException("Update delegation key is not a valid use case for stateless secret managers"); + } + + private void storeNewToken(RMDelegationTokenIdentifier identifier, DelegationTokenInformation tokenInfo) throws YarnException, IOException { + long renewDate = tokenInfo.getRenewDate(); + String token = RouterDelegationTokenSupport.encodeDelegationTokenInformation(tokenInfo); + federationFacade.storeNewToken(identifier, renewDate, token); + } + + private void updateStoredToken(RMDelegationTokenIdentifier identifier, DelegationTokenInformation tokenInfo) throws YarnException, IOException { + long renewDate = tokenInfo.getRenewDate(); + String token = RouterDelegationTokenSupport.encodeDelegationTokenInformation(tokenInfo); + federationFacade.updateStoredToken(identifier, renewDate, token); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java index 6f7248a0866b4..94d893929e633 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java @@ -1618,7 +1618,7 @@ public void testGetDelegationToken() throws IOException, YarnException { RouterRMDTSecretManagerState managerState = stateStore.getRouterRMSecretManagerState(); Assert.assertNotNull(managerState); - Map delegationTokenState = + Map delegationTokenState = managerState.getTokenState(); Assert.assertNotNull(delegationTokenState); Assert.assertTrue(delegationTokenState.containsKey(rMDelegationTokenIdentifier)); @@ -1671,7 +1671,7 @@ public void testRenewDelegationToken() throws IOException, YarnException { // Step3. Compare whether the expirationTime returned to // the client is consistent with the renewDate in the stateStore RouterRMDTSecretManagerState managerState = stateStore.getRouterRMSecretManagerState(); - Map delegationTokenState = + Map delegationTokenState = managerState.getTokenState(); Assert.assertNotNull(delegationTokenState); Assert.assertTrue(delegationTokenState.containsKey(rMDelegationTokenIdentifier)); @@ -1707,7 +1707,7 @@ public void testCancelDelegationToken() throws IOException, YarnException { // Step3. Query the data in the StateStore and confirm that the Delegation has been deleted. // At this point, the size of delegationTokenState should be 0. RouterRMDTSecretManagerState managerState = stateStore.getRouterRMSecretManagerState(); - Map delegationTokenState = + Map delegationTokenState = managerState.getTokenState(); Assert.assertNotNull(delegationTokenState); Assert.assertEquals(0, delegationTokenState.size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java deleted file mode 100644 index eac2c5a03ba61..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java +++ /dev/null @@ -1,201 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.router.secure; - -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.token.delegation.DelegationKey; -import org.apache.hadoop.test.LambdaTestUtils; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; -import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; -import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertArrayEquals; - -public class TestRouterDelegationTokenSecretManager extends AbstractSecureRouterTest { - - private static final Logger LOG = - LoggerFactory.getLogger(TestRouterDelegationTokenSecretManager.class); - - @Test - public void testRouterStoreNewMasterKey() throws Exception { - LOG.info("Test RouterDelegationTokenSecretManager: StoreNewMasterKey."); - - // Start the Router in Secure Mode - startSecureRouter(); - - // Store NewMasterKey - RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = - routerClientRMService.getRouterDTSecretManager(); - DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); - secretManager.storeNewMasterKey(storeKey); - - // Get DelegationKey - DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); - DelegationKey responseKey = secretManager.getMasterKeyByDelegationKey(paramKey); - - assertNotNull(paramKey); - assertEquals(storeKey.getExpiryDate(), responseKey.getExpiryDate()); - assertEquals(storeKey.getKeyId(), responseKey.getKeyId()); - assertArrayEquals(storeKey.getEncodedKey(), responseKey.getEncodedKey()); - assertEquals(storeKey, responseKey); - - stopSecureRouter(); - } - - @Test - public void testRouterRemoveStoredMasterKey() throws Exception { - LOG.info("Test RouterDelegationTokenSecretManager: RemoveStoredMasterKey."); - - // Start the Router in Secure Mode - startSecureRouter(); - - // Store NewMasterKey - RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = - routerClientRMService.getRouterDTSecretManager(); - DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); - secretManager.storeNewMasterKey(storeKey); - - // Remove DelegationKey - secretManager.removeStoredMasterKey(storeKey); - - // Get DelegationKey - DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); - LambdaTestUtils.intercept(IOException.class, - "GetMasterKey with keyID: " + storeKey.getKeyId() + " does not exist.", - () -> secretManager.getMasterKeyByDelegationKey(paramKey)); - - stopSecureRouter(); - } - - @Test - public void testRouterStoreNewToken() throws Exception { - LOG.info("Test RouterDelegationTokenSecretManager: StoreNewToken."); - - // Start the Router in Secure Mode - startSecureRouter(); - - // Store new rm-token - RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = - routerClientRMService.getRouterDTSecretManager(); - RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( - new Text("owner1"), new Text("renewer1"), new Text("realuser1")); - int sequenceNumber = 1; - dtId1.setSequenceNumber(sequenceNumber); - Long renewDate1 = Time.now(); - secretManager.storeNewToken(dtId1, renewDate1); - - // query rm-token - RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier( - new Text("owner1"), new Text("renewer1"), new Text("realuser1")); - dtId2.setSequenceNumber(sequenceNumber); - RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2); - Assert.assertEquals(dtId1, dtId3); - - // query rm-token2 not exists - sequenceNumber++; - dtId2.setSequenceNumber(2); - LambdaTestUtils.intercept(YarnException.class, - "RMDelegationToken: " + dtId2 + " does not exist.", - () -> secretManager.getTokenByRouterStoreToken(dtId2)); - - stopSecureRouter(); - } - - @Test - public void testRouterUpdateNewToken() throws Exception { - LOG.info("Test RouterDelegationTokenSecretManager: UpdateNewToken."); - - // Start the Router in Secure Mode - startSecureRouter(); - - // Store new rm-token - RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = - routerClientRMService.getRouterDTSecretManager(); - RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( - new Text("owner1"), new Text("renewer1"), new Text("realuser1")); - int sequenceNumber = 1; - dtId1.setSequenceNumber(sequenceNumber); - Long renewDate1 = Time.now(); - secretManager.storeNewToken(dtId1, renewDate1); - - sequenceNumber++; - dtId1.setSequenceNumber(sequenceNumber); - secretManager.updateStoredToken(dtId1, renewDate1); - - // query rm-token - RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier( - new Text("owner1"), new Text("renewer1"), new Text("realuser1")); - dtId2.setSequenceNumber(sequenceNumber); - RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2); - assertNotNull(dtId3); - assertEquals(dtId1.getKind(), dtId3.getKind()); - assertEquals(dtId1.getOwner(), dtId3.getOwner()); - assertEquals(dtId1.getRealUser(), dtId3.getRealUser()); - assertEquals(dtId1.getRenewer(), dtId3.getRenewer()); - assertEquals(dtId1.getIssueDate(), dtId3.getIssueDate()); - assertEquals(dtId1.getMasterKeyId(), dtId3.getMasterKeyId()); - assertEquals(dtId1.getSequenceNumber(), dtId3.getSequenceNumber()); - assertEquals(sequenceNumber, dtId3.getSequenceNumber()); - assertEquals(dtId1, dtId3); - - stopSecureRouter(); - } - - @Test - public void testRouterRemoveToken() throws Exception { - LOG.info("Test RouterDelegationTokenSecretManager: RouterRemoveToken."); - - // Start the Router in Secure Mode - startSecureRouter(); - - // Store new rm-token - RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); - RouterDelegationTokenSecretManager secretManager = - routerClientRMService.getRouterDTSecretManager(); - RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( - new Text("owner1"), new Text("renewer1"), new Text("realuser1")); - int sequenceNumber = 1; - dtId1.setSequenceNumber(sequenceNumber); - Long renewDate1 = Time.now(); - secretManager.storeNewToken(dtId1, renewDate1); - - // Remove rm-token - secretManager.removeStoredToken(dtId1); - - // query rm-token - LambdaTestUtils.intercept(YarnException.class, - "RMDelegationToken: " + dtId1 + " does not exist.", - () -> secretManager.getTokenByRouterStoreToken(dtId1)); - - stopSecureRouter(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/AbstractSecureRouterTest.java similarity index 99% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/AbstractSecureRouterTest.java index 062d732e8738b..0ea0c3ec82c4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/AbstractSecureRouterTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.router.secure; +package org.apache.hadoop.yarn.server.router.security; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/TestRouterDelegationTokenSecretManager.java new file mode 100644 index 0000000000000..aae52b632b199 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/TestRouterDelegationTokenSecretManager.java @@ -0,0 +1,251 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.security; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.mockito.Mockito; + +import java.io.IOException; + +public class TestRouterDelegationTokenSecretManager extends AbstractSecureRouterTest { + + private volatile RouterDelegationTokenSecretManager secretManager_1; + private volatile RouterDelegationTokenSecretManager secretManager_2; + private final Text owner = new Text("owner"); + private final Text renewer = new Text("renewer"); + private final Text realUser = new Text("realUser"); + private final int keyUpdateInterval = 1000; + private final int tokenRenewInterval = 2000; + private final int tokenMaxLifeTime = 10000; + + @Before + public void setup() { + + // Setup multiple secret managers to validate stateless secret managers. + // They are using same instance of FederationStateStoreFacade thus the in memory state store is shared + secretManager_1 = Mockito.spy(new RouterDelegationTokenSecretManager( + keyUpdateInterval, tokenMaxLifeTime, tokenRenewInterval, 100) + ); + secretManager_2 = Mockito.spy(new RouterDelegationTokenSecretManager( + keyUpdateInterval, tokenMaxLifeTime, tokenRenewInterval, 100) + ); + } + + @After + public void cleanup() throws Exception { + secretManager_1.stopThreads(); + secretManager_2.stopThreads(); + secretManager_1 = null; + secretManager_2 = null; + FederationStateStoreFacade.getInstance().getStateStore().close(); + } + + @Test + public void testNewTokenIsVerifiedAcrossManagers() throws IOException { + + secretManager_1.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(owner, renewer, realUser); + Token token = new Token<>(tokenIdentifier, secretManager_1); + + Token token2 = new Token<>(); + token2.decodeFromUrlString(token.encodeToUrlString()); + + RMDelegationTokenIdentifier tokenIdentifier_2 = secretManager_1.decodeTokenIdentifier(token2); + Assertions.assertDoesNotThrow(() -> secretManager_1.verifyToken(tokenIdentifier_2, token2.getPassword())); + + secretManager_2.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier_3 = secretManager_2.decodeTokenIdentifier(token2); + Assertions.assertDoesNotThrow(() -> secretManager_2.verifyToken(tokenIdentifier_3, token2.getPassword())); + } + + @Test + public void testMasterKeyIsRolled() throws IOException, InterruptedException { + + secretManager_1.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier1 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + new Token<>(tokenIdentifier1, secretManager_1); + + RMDelegationTokenIdentifier tokenIdentifier2 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + new Token<>(tokenIdentifier2, secretManager_1); + + // Check multiple tokens have same master key + Assert.assertEquals(tokenIdentifier1.getMasterKeyId(), tokenIdentifier2.getMasterKeyId()); + // Sleep until master key is updated + Thread.sleep(keyUpdateInterval + 100); + + RMDelegationTokenIdentifier tokenIdentifier3 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + new Token<>(tokenIdentifier3, secretManager_1); + // Check master key is updated + Assert.assertNotEquals(tokenIdentifier1.getMasterKeyId(), tokenIdentifier3.getMasterKeyId()); + + } + + @Test + public void testNewTokenIsCancelledAcrossManagers() throws IOException { + + secretManager_1.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(owner, renewer, realUser); + Token token = new Token<>(tokenIdentifier, secretManager_1); + + Token token2 = new Token<>(); + token2.decodeFromUrlString(token.encodeToUrlString()); + + secretManager_2.startThreads(); + secretManager_2.cancelToken(token2, owner.toString()); + + RMDelegationTokenIdentifier tokenIdentifier_2 = secretManager_1.decodeTokenIdentifier(token2); + Assertions.assertThrows(SecretManager.InvalidToken.class, + () -> secretManager_1.verifyToken(tokenIdentifier_2, token2.getPassword()) + ); + + } + + @Test + public void testNewTokenIsRenewedAcrossManagers() throws IOException, InterruptedException { + + secretManager_1.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(owner, renewer, realUser); + Token token = new Token<>(tokenIdentifier, secretManager_1); + + Token token2 = new Token<>(); + token2.decodeFromUrlString(token.encodeToUrlString()); + + Thread.sleep(tokenRenewInterval / 2 + 100); + secretManager_2.startThreads(); + secretManager_2.renewToken(token2, renewer.toString()); + + Thread.sleep(tokenRenewInterval / 2 + 100); + RMDelegationTokenIdentifier tokenIdentifier_2 = secretManager_1.decodeTokenIdentifier(token2); + Assertions.assertDoesNotThrow(() -> secretManager_1.verifyToken(tokenIdentifier_2, token2.getPassword())); + + } + + @Test + public void testTokenOperationsOnMasterKeyRollover() throws IOException, InterruptedException { + + secretManager_1.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier1 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + Token token1 = new Token<>(tokenIdentifier1, secretManager_1); + + // Sleep until master key is updated + Thread.sleep(keyUpdateInterval + 100); + + RMDelegationTokenIdentifier tokenIdentifier2 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + new Token<>(tokenIdentifier2, secretManager_1); + // Check master key is updated + Assert.assertNotEquals(tokenIdentifier1.getMasterKeyId(), tokenIdentifier2.getMasterKeyId()); + + // Verify token with old master key is still considered valid + Assertions.assertDoesNotThrow(() -> secretManager_1.verifyToken(tokenIdentifier1, token1.getPassword())); + // Verify token with old master key can be renewed + Assertions.assertDoesNotThrow(() -> secretManager_1.renewToken(token1, renewer.toString())); + // Verify token with old master key can be cancelled + Assertions.assertDoesNotThrow(() -> secretManager_1.cancelToken(token1, owner.toString())); + // Verify token with old master key is now cancelled + Assert.assertThrows(SecretManager.InvalidToken.class, + () -> secretManager_1.verifyToken(tokenIdentifier1, token1.getPassword())); + + } + + @Test + public void testMasterKeyIsNotRolledOver() throws IOException, InterruptedException { + + secretManager_1.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier1 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + new Token<>(tokenIdentifier1, secretManager_1); + + Mockito.doThrow(new IOException("failure")).when(secretManager_1).storeNewMasterKey(Mockito.any()); + + // Sleep until master key is updated + Thread.sleep(keyUpdateInterval + 100); + + RMDelegationTokenIdentifier tokenIdentifier2 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + new Token<>(tokenIdentifier2, secretManager_1); + // Verify master key is not updated + Assert.assertEquals(tokenIdentifier1.getMasterKeyId(), tokenIdentifier2.getMasterKeyId()); + } + + @Test + public void testNewTokenFailsOnDBFailure() throws IOException { + secretManager_1.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier1 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + + Mockito.doThrow(new IOException("failure")).when(secretManager_1).storeToken(Mockito.any(), Mockito.any()); + Assert.assertThrows(RuntimeException.class, () -> new Token<>(tokenIdentifier1, secretManager_1)); + } + + @Test + public void testTokenIsNotRenewedOnDBFailure() throws IOException, InterruptedException { + secretManager_1.startThreads(); + RMDelegationTokenIdentifier tokenIdentifier1 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + + Token token = new Token<>(tokenIdentifier1, secretManager_1); + Mockito.doThrow(new IOException("failure")).when(secretManager_1).updateToken(Mockito.any(), Mockito.any()); + + Thread.sleep(tokenRenewInterval / 2 + 100); + Assert.assertThrows(IOException.class, () -> secretManager_1.renewToken(token, renewer.toString())); + // validate that token is currently valid + Assertions.assertDoesNotThrow(() -> secretManager_1.verifyToken(tokenIdentifier1, token.getPassword())); + + Thread.sleep(tokenRenewInterval / 2 + 100); + // token is no longer valid because token renewal had failed + Assertions.assertThrows(SecretManager.InvalidToken.class, + () -> secretManager_1.verifyToken(tokenIdentifier1, token.getPassword()) + ); + } + + @Ignore + public void testNewTokenFailureIfMasterKeyNotRolledOverAtAll() throws IOException, InterruptedException { + secretManager_1.startThreads(); + + // Token generation succeeds initially because master key generated on initialisation was saved + RMDelegationTokenIdentifier tokenIdentifier1 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + new Token<>(tokenIdentifier1, secretManager_1); + + Mockito.doThrow(new IOException("failure")).when(secretManager_1).storeNewMasterKey(Mockito.any()); + + // Sleep until current master key expires. New master key isn't generated because rollovers are failing + Thread.sleep(tokenMaxLifeTime + keyUpdateInterval + 100); + + RMDelegationTokenIdentifier tokenIdentifier2 = new RMDelegationTokenIdentifier(owner, renewer, realUser); + // New token generation fails because current master key is expired + Assert.assertThrows(RuntimeException.class, () -> new Token<>(tokenIdentifier2, secretManager_1)); + } + + @Test + public void testMasterKeyCreationFailureOnStartup() throws IOException { + Mockito.doThrow(new IOException("failure")).when(secretManager_1).storeNewMasterKey(Mockito.any()); + + Assert.assertThrows(IOException.class, () -> secretManager_1.startThreads()); + + RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(owner, renewer, realUser); + // New token generation fails because master key is not yet set + Assert.assertThrows(NullPointerException.class, () -> new Token<>(tokenIdentifier, secretManager_1)); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/TestSecureLogins.java similarity index 99% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/TestSecureLogins.java index 40911814c044c..ac22d1073a32b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/security/TestSecureLogins.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.router.secure; +package org.apache.hadoop.yarn.server.router.security; import org.apache.commons.collections.MapUtils; import org.apache.hadoop.service.Service;