Skip to content

Commit

Permalink
Adding counter to track token failures separately
Browse files Browse the repository at this point in the history
  • Loading branch information
@hchaverri
hchaverri committed Mar 25, 2022
1 parent 2c7ac32 commit 9b91215
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 13 deletions.
56 changes: 43 additions & 13 deletions ...ava/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.metrics2.util.Metrics2Util.NameValuePair;
import org.apache.hadoop.metrics2.util.Metrics2Util.TopN;
Expand Down Expand Up @@ -446,14 +447,16 @@ protected synchronized byte[] createPassword(TokenIdent identifier) {
DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now
+ tokenRenewInterval, password, getTrackingIdIfEnabled(identifier));
long start = Time.monotonicNow();
boolean success = false;
try {
storeToken(identifier, tokenInfo);
success = true;
} catch (IOException ioe) {
LOG.error("Could not store token " + formatTokenId(identifier) + "!!",
ioe);
} finally {
if (metrics != null) {
metrics.addStoreToken(Time.monotonicNow() - start);
metrics.addStoreToken(success, Time.monotonicNow() - start);
}
}
return password;
Expand Down Expand Up @@ -577,11 +580,13 @@ public synchronized long renewToken(Token<TokenIdent> token,
+ formatTokenId(id));
}
long start = Time.monotonicNow();
boolean success = false;
try {
updateToken(id, info);
success = true;
} finally {
if (metrics != null) {
metrics.addUpdateToken(Time.monotonicNow() - start);
metrics.addUpdateToken(success, Time.monotonicNow() - start);
}
}
return renewTime;
Expand Down Expand Up @@ -620,12 +625,14 @@ public synchronized TokenIdent cancelToken(Token<TokenIdent> token,
throw new InvalidToken("Token not found " + formatTokenId(id));
}
long start = Time.monotonicNow();
boolean success = false;
try {
removeTokenForOwnerStats(id);
removeStoredToken(id);
success = true;
} finally {
if (metrics != null) {
metrics.addRemoveToken(Time.monotonicNow() - start);
metrics.addRemoveToken(success, Time.monotonicNow() - start);
}
}
return id;
Expand Down Expand Up @@ -867,11 +874,14 @@ protected void syncTokenOwnerStats() {
*/
@Metrics(about="Delegation token secret manager metrics", context="token")
static class DelegationTokenSecretManagerMetrics implements IOStatisticsSource {
private static final Logger LOG = LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class);

final static String STORE_TOKEN_STAT = "storeToken";
final static String UPDATE_TOKEN_STAT = "updateToken";
final static String REMOVE_TOKEN_STAT = "removeToken";
final static String TOKEN_FAILURE_STAT = "tokenFailure";

final MetricsRegistry registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics");
final MetricsRegistry registry;
final IOStatisticsStore ioStatistics;

@Metric("Rate of storage of delegation tokens and latency (milliseconds)")
Expand All @@ -880,6 +890,8 @@ static class DelegationTokenSecretManagerMetrics implements IOStatisticsSource {
MutableRate updateToken;
@Metric("Rate of removal of delegation tokens and latency (milliseconds)")
MutableRate removeToken;
@Metric("Counter of delegation tokens operation failures")
MutableCounterLong tokenFailure;

static DelegationTokenSecretManagerMetrics create() {
return DefaultMetricsSystem.instance().register(new DelegationTokenSecretManagerMetrics());
Expand All @@ -888,22 +900,40 @@ static DelegationTokenSecretManagerMetrics create() {
public DelegationTokenSecretManagerMetrics() {
ioStatistics = iostatisticsStore()
.withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT, REMOVE_TOKEN_STAT)
.withCounters(TOKEN_FAILURE_STAT)
.build();
registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics");
LOG.debug("Initialized {}", registry);
}

public void addStoreToken(long value) {
storeToken.add(value);
ioStatistics.addTimedOperation(STORE_TOKEN_STAT, value);
public void addStoreToken(boolean success, long value) {
if (success) {
storeToken.add(value);
ioStatistics.addTimedOperation(STORE_TOKEN_STAT, value);
} else {
tokenFailure.incr();
ioStatistics.incrementCounter(TOKEN_FAILURE_STAT);
}
}

public void addUpdateToken(long value) {
updateToken.add(value);
ioStatistics.addTimedOperation(UPDATE_TOKEN_STAT, value);
public void addUpdateToken(boolean success, long value) {
if (success) {
updateToken.add(value);
ioStatistics.addTimedOperation(UPDATE_TOKEN_STAT, value);
} else {
tokenFailure.incr();
ioStatistics.incrementCounter(TOKEN_FAILURE_STAT);
}
}

public void addRemoveToken(long value) {
removeToken.add(value);
ioStatistics.addTimedOperation(REMOVE_TOKEN_STAT, value);
public void addRemoveToken(boolean success, long value) {
if (success) {
removeToken.add(value);
ioStatistics.addTimedOperation(REMOVE_TOKEN_STAT, value);
} else {
tokenFailure.incr();
ioStatistics.incrementCounter(TOKEN_FAILURE_STAT);
}
}

@Override
Expand Down
72 changes: 72 additions & 0 deletions ...common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,42 @@ public DelegationKey getKey(TestDelegationTokenIdentifier id) {
return allKeys.get(id.getMasterKeyId());
}
}

public static class TestFailureDelegationTokenSecretManager extends TestDelegationTokenSecretManager {
private boolean throwError = false;

public TestFailureDelegationTokenSecretManager() {
super(24*60*60*1000, 10*1000, 1*1000, 3600000);
}

public void setThrowError(boolean throwError) {
this.throwError = throwError;
}

@Override
protected void storeNewToken(TestDelegationTokenIdentifier ident, long renewDate) throws IOException {
if (throwError) {
throw new IOException("Test exception");
}
super.storeNewToken(ident, renewDate);
}

@Override
protected void removeStoredToken(TestDelegationTokenIdentifier ident) throws IOException {
if (throwError) {
throw new IOException("Test exception");
}
super.removeStoredToken(ident);
}

@Override
protected void updateStoredToken(TestDelegationTokenIdentifier ident, long renewDate) throws IOException {
if (throwError) {
throw new IOException("Test exception");
}
super.updateStoredToken(ident, renewDate);
}
}

public static class TokenSelector extends
AbstractDelegationTokenSelector<TestDelegationTokenIdentifier>{
Expand Down Expand Up @@ -604,4 +640,40 @@ public void testDelegationTokenSecretManagerMetrics() throws Exception {
dtSecretManager.stopThreads();
}
}

@Test
public void testDelegationTokenSecretManagerMetricsFailures() throws Exception {
TestFailureDelegationTokenSecretManager dtSecretManager = new TestFailureDelegationTokenSecretManager();

try {
dtSecretManager.startThreads();

final Token<TestDelegationTokenIdentifier> token =
generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker");

dtSecretManager.setThrowError(true);

Assert.assertEquals(0, dtSecretManager.metrics.tokenFailure.value());
generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker");
Assert.assertEquals(1, dtSecretManager.metrics.tokenFailure.value());

try {
dtSecretManager.renewToken(token, "JobTracker");
Assert.fail("Expected exception");
} catch (Exception ex) {
// Expected exception
}
Assert.assertEquals(2, dtSecretManager.metrics.tokenFailure.value());

try {
dtSecretManager.cancelToken(token, "JobTracker");
Assert.fail("Expected exception");
} catch (Exception ex) {
// Expected exception
}
Assert.assertEquals(3, dtSecretManager.metrics.tokenFailure.value());
} finally {
dtSecretManager.stopThreads();
}
}
}

0 comments on commit 9b91215

Please sign in to comment.