From b532058f1f57d8e48d892f7429eb1b6cf0e0f926 Mon Sep 17 00:00:00 2001 From: Victor Li Date: Mon, 13 Feb 2023 16:23:46 -0800 Subject: [PATCH 1/5] HBASE-15242: add client side metrics for timeout and remote exceptions. --- .../hbase/client/MetricsConnection.java | 16 ++++- .../hadoop/hbase/ipc/AbstractRpcClient.java | 7 +- .../hbase/client/TestMetricsConnection.java | 67 ++++++++++++++++--- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java index 75dc7aea27ce..c82c23e1703f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java @@ -36,6 +36,8 @@ import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ipc.CallTimeoutException; +import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; @@ -118,6 +120,9 @@ static String getScope(Configuration conf, String clusterId, Object connectionOb private static final String CNT_BASE = "rpcCount_"; private static final String FAILURE_CNT_BASE = "rpcFailureCount_"; + private static final String TIMEOUT_CNT_BASE = "rpcExceptionCallTimeout_"; + private static final String REMOTE_CNT_BASE = "rpcExceptionRemote_"; + private static final String OTHERS_CNT_BASE = "rpcExceptionOthers_"; private static final String DRTN_BASE = "rpcCallDurationMs_"; private static final String REQ_BASE = "rpcCallRequestSizeBytes_"; private static final String RESP_BASE = "rpcCallResponseSizeBytes_"; @@ -638,7 +643,7 @@ private void shutdown() { } /** Report RPC context to metrics system. */ - public void updateRpc(MethodDescriptor method, Message param, CallStats stats, boolean failed) { + public void updateRpc(MethodDescriptor method, Message param, CallStats stats, Throwable e) { int callsPerServer = stats.getConcurrentCallsPerServer(); if (callsPerServer > 0) { concurrentCallsPerServerHist.update(callsPerServer); @@ -646,8 +651,15 @@ public void updateRpc(MethodDescriptor method, Message param, CallStats stats, b // Update the counter that tracks RPCs by type. final String methodName = method.getService().getName() + "_" + method.getName(); getMetric(CNT_BASE + methodName, rpcCounters, counterFactory).inc(); - if (failed) { + if (e != null) { getMetric(FAILURE_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); + if (e instanceof CallTimeoutException) { + getMetric(TIMEOUT_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); + } else if (e instanceof RemoteWithExtrasException) { + getMetric(REMOTE_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); + } else { + getMetric(OTHERS_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); + } } // this implementation is tied directly to protobuf implementation details. would be better // if we could dispatch based on something static, ie, request Message type. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java index 8b56406ed58e..23d14c272d2b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java @@ -375,16 +375,15 @@ private T getConnection(ConnectionId remoteId) throws IOException { private void onCallFinished(Call call, HBaseRpcController hrc, Address addr, RpcCallback callback) { call.callStats.setCallTimeMs(EnvironmentEdgeManager.currentTime() - call.getStartTime()); - final boolean failed = (call.error != null) ? true : false; if (metrics != null) { - metrics.updateRpc(call.md, call.param, call.callStats, failed); + metrics.updateRpc(call.md, call.param, call.callStats, call.error); } if (LOG.isTraceEnabled()) { LOG.trace("CallId: {}, call: {}, startTime: {}ms, callTime: {}ms, status: {}", call.id, call.md.getName(), call.getStartTime(), call.callStats.getCallTimeMs(), - failed ? "failed" : "successful"); + call.error != null ? "failed" : "successful"); } - if (failed) { + if (call.error != null) { if (call.error instanceof RemoteException) { call.error.fillInStackTrace(); hrc.setFailed(call.error); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java index edc846754f6f..cc70225041c6 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java @@ -31,6 +31,8 @@ import java.util.concurrent.ThreadPoolExecutor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ipc.CallTimeoutException; +import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MetricsTests; @@ -150,51 +152,98 @@ public void testStaticMetrics() throws IOException { for (int i = 0; i < loop; i++) { METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Get"), - GetRequest.getDefaultInstance(), MetricsConnection.newCallStats(), false); + GetRequest.getDefaultInstance(), MetricsConnection.newCallStats(), null); METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Scan"), - ScanRequest.getDefaultInstance(), MetricsConnection.newCallStats(), false); + ScanRequest.getDefaultInstance(), MetricsConnection.newCallStats(), + new RemoteWithExtrasException("IOException", null, false, false)); METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Multi"), - MultiRequest.getDefaultInstance(), MetricsConnection.newCallStats(), true); + MultiRequest.getDefaultInstance(), MetricsConnection.newCallStats(), + new CallTimeoutException("test with CallTimeoutException")); METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Mutate"), MutateRequest.newBuilder() .setMutation(ProtobufUtil.toMutation(MutationType.APPEND, new Append(foo))) .setRegion(region).build(), - MetricsConnection.newCallStats(), false); + MetricsConnection.newCallStats(), null); METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Mutate"), MutateRequest.newBuilder() .setMutation(ProtobufUtil.toMutation(MutationType.DELETE, new Delete(foo))) .setRegion(region).build(), - MetricsConnection.newCallStats(), false); + MetricsConnection.newCallStats(), null); METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Mutate"), MutateRequest.newBuilder() .setMutation(ProtobufUtil.toMutation(MutationType.INCREMENT, new Increment(foo))) .setRegion(region).build(), - MetricsConnection.newCallStats(), false); + MetricsConnection.newCallStats(), null); METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Mutate"), MutateRequest.newBuilder() .setMutation(ProtobufUtil.toMutation(MutationType.PUT, new Put(foo))).setRegion(region) .build(), - MetricsConnection.newCallStats(), false); + MetricsConnection.newCallStats(), null); } + final String rpcCountPrefix = "rpcCount_" + ClientService.getDescriptor().getName() + "_"; final String rpcFailureCountPrefix = "rpcFailureCount_" + ClientService.getDescriptor().getName() + "_"; + final String rpcTimeoutCountPrefix = + "rpcExceptionCallTimeout_" + ClientService.getDescriptor().getName() + "_"; + final String rpcRemoteCountPrefix = + "rpcExceptionRemote_" + ClientService.getDescriptor().getName() + "_"; String metricKey; long metricVal; Counter counter; - for (String method : new String[] { "Get", "Scan", "Mutate" }) { + + for (String method : new String[] { "Get", "Mutate" }) { metricKey = rpcCountPrefix + method; metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop); + + // no failure metricKey = rpcFailureCountPrefix + method; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0); } - metricKey = rpcFailureCountPrefix + "Multi"; + + final String scanMethod = "Scan"; + metricKey = rpcCountPrefix + scanMethod; + metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop); + // has failure + metricKey = rpcFailureCountPrefix + scanMethod; + counter = METRICS.getRpcCounters().get(metricKey); + metricVal = (counter != null) ? counter.getCount() : 0; + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + // no timeout + metricKey = rpcTimeoutCountPrefix + scanMethod; + counter = METRICS.getRpcCounters().get(metricKey); + metricVal = (counter != null) ? counter.getCount() : 0; + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0); + // has remote + metricKey = rpcRemoteCountPrefix + scanMethod; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + + final String multiMethod = "Multi"; + metricKey = rpcCountPrefix + multiMethod; + metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop); + // has failure + metricKey = rpcFailureCountPrefix + multiMethod; + counter = METRICS.getRpcCounters().get(metricKey); + metricVal = (counter != null) ? counter.getCount() : 0; + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + // has timeout + metricKey = rpcTimeoutCountPrefix + multiMethod; + counter = METRICS.getRpcCounters().get(metricKey); + metricVal = (counter != null) ? counter.getCount() : 0; + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + // no remote + metricKey = rpcRemoteCountPrefix + multiMethod; + counter = METRICS.getRpcCounters().get(metricKey); + metricVal = (counter != null) ? counter.getCount() : 0; + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0); + for (MetricsConnection.CallTracker t : new MetricsConnection.CallTracker[] { METRICS.getGetTracker(), METRICS.getScanTracker(), METRICS.getMultiTracker(), METRICS.getAppendTracker(), METRICS.getDeleteTracker(), METRICS.getIncrementTracker(), From 76ee7b3085c52a17e7613e1cb80107e5f2dae5bc Mon Sep 17 00:00:00 2001 From: Victor Li Date: Tue, 21 Feb 2023 10:33:04 -0800 Subject: [PATCH 2/5] Instrument all exceptions separatedly. --- .../hbase/client/MetricsConnection.java | 15 +++--- .../hbase/client/TestMetricsConnection.java | 51 +++++-------------- 2 files changed, 19 insertions(+), 47 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java index c82c23e1703f..4461e117374e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java @@ -120,9 +120,8 @@ static String getScope(Configuration conf, String clusterId, Object connectionOb private static final String CNT_BASE = "rpcCount_"; private static final String FAILURE_CNT_BASE = "rpcFailureCount_"; - private static final String TIMEOUT_CNT_BASE = "rpcExceptionCallTimeout_"; - private static final String REMOTE_CNT_BASE = "rpcExceptionRemote_"; - private static final String OTHERS_CNT_BASE = "rpcExceptionOthers_"; + private static final String LOCAL_EXCEPTION_CNT_BASE = "rpcLocalExceptions_"; + private static final String REMOTE_EXCEPTION_CNT_BASE = "rpcRemoteExceptions_"; private static final String DRTN_BASE = "rpcCallDurationMs_"; private static final String REQ_BASE = "rpcCallRequestSizeBytes_"; private static final String RESP_BASE = "rpcCallResponseSizeBytes_"; @@ -653,12 +652,12 @@ public void updateRpc(MethodDescriptor method, Message param, CallStats stats, T getMetric(CNT_BASE + methodName, rpcCounters, counterFactory).inc(); if (e != null) { getMetric(FAILURE_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); - if (e instanceof CallTimeoutException) { - getMetric(TIMEOUT_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); - } else if (e instanceof RemoteWithExtrasException) { - getMetric(REMOTE_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); + if (e instanceof RemoteWithExtrasException) { + getMetric(REMOTE_EXCEPTION_CNT_BASE + ((RemoteWithExtrasException)e).getClassName(), + rpcCounters, counterFactory).inc(); } else { - getMetric(OTHERS_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); + getMetric(LOCAL_EXCEPTION_CNT_BASE + e.getClass().getSimpleName(), + rpcCounters, counterFactory).inc(); } } // this implementation is tied directly to protobuf implementation details. would be better diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java index cc70225041c6..fff19805bb20 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java @@ -184,65 +184,38 @@ public void testStaticMetrics() throws IOException { final String rpcCountPrefix = "rpcCount_" + ClientService.getDescriptor().getName() + "_"; final String rpcFailureCountPrefix = "rpcFailureCount_" + ClientService.getDescriptor().getName() + "_"; - final String rpcTimeoutCountPrefix = - "rpcExceptionCallTimeout_" + ClientService.getDescriptor().getName() + "_"; - final String rpcRemoteCountPrefix = - "rpcExceptionRemote_" + ClientService.getDescriptor().getName() + "_"; String metricKey; long metricVal; Counter counter; - for (String method : new String[] { "Get", "Mutate" }) { + for (String method : new String[] { "Get", "Scan", "Multi", "Mutate" }) { metricKey = rpcCountPrefix + method; metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop); - // no failure metricKey = rpcFailureCountPrefix + method; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0); + if (method.equals("Get") || method.equals("Mutate")) { + // no failure + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0); + } else { + // has failure + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + } } - final String scanMethod = "Scan"; - metricKey = rpcCountPrefix + scanMethod; - metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop); - // has failure - metricKey = rpcFailureCountPrefix + scanMethod; - counter = METRICS.getRpcCounters().get(metricKey); - metricVal = (counter != null) ? counter.getCount() : 0; - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); - // no timeout - metricKey = rpcTimeoutCountPrefix + scanMethod; - counter = METRICS.getRpcCounters().get(metricKey); - metricVal = (counter != null) ? counter.getCount() : 0; - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0); - // has remote - metricKey = rpcRemoteCountPrefix + scanMethod; + // remote exception + metricKey = "rpcRemoteExceptions_IOException"; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); - final String multiMethod = "Multi"; - metricKey = rpcCountPrefix + multiMethod; - metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop); - // has failure - metricKey = rpcFailureCountPrefix + multiMethod; + // local exception + metricKey = "rpcLocalExceptions_CallTimeoutException"; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); - // has timeout - metricKey = rpcTimeoutCountPrefix + multiMethod; - counter = METRICS.getRpcCounters().get(metricKey); - metricVal = (counter != null) ? counter.getCount() : 0; - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); - // no remote - metricKey = rpcRemoteCountPrefix + multiMethod; - counter = METRICS.getRpcCounters().get(metricKey); - metricVal = (counter != null) ? counter.getCount() : 0; - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0); for (MetricsConnection.CallTracker t : new MetricsConnection.CallTracker[] { METRICS.getGetTracker(), METRICS.getScanTracker(), METRICS.getMultiTracker(), From cbcc335b3558d2c16835d76d965889564f557013 Mon Sep 17 00:00:00 2001 From: Victor Li Date: Tue, 21 Feb 2023 10:44:38 -0800 Subject: [PATCH 3/5] cosmetics changes. --- .../java/org/apache/hadoop/hbase/client/MetricsConnection.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java index 4461e117374e..4104a12f6ca8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java @@ -36,7 +36,6 @@ import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.ipc.CallTimeoutException; import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; From 4e74167969d6fd47c53f7731db1e4f9950463181 Mon Sep 17 00:00:00 2001 From: Victor Li Date: Tue, 21 Feb 2023 10:46:10 -0800 Subject: [PATCH 4/5] fix spotless. --- .../org/apache/hadoop/hbase/client/MetricsConnection.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java index 4104a12f6ca8..9ba8dc100157 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java @@ -652,11 +652,11 @@ public void updateRpc(MethodDescriptor method, Message param, CallStats stats, T if (e != null) { getMetric(FAILURE_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); if (e instanceof RemoteWithExtrasException) { - getMetric(REMOTE_EXCEPTION_CNT_BASE + ((RemoteWithExtrasException)e).getClassName(), + getMetric(REMOTE_EXCEPTION_CNT_BASE + ((RemoteWithExtrasException) e).getClassName(), rpcCounters, counterFactory).inc(); } else { - getMetric(LOCAL_EXCEPTION_CNT_BASE + e.getClass().getSimpleName(), - rpcCounters, counterFactory).inc(); + getMetric(LOCAL_EXCEPTION_CNT_BASE + e.getClass().getSimpleName(), rpcCounters, + counterFactory).inc(); } } // this implementation is tied directly to protobuf implementation details. would be better From b1bf32ffdbeab80b47d20d2c7c8798009db6f487 Mon Sep 17 00:00:00 2001 From: Victor Li Date: Tue, 21 Feb 2023 13:47:05 -0800 Subject: [PATCH 5/5] Fixed exception metrics with simple class name for remote exceptions. --- .../hadoop/hbase/client/MetricsConnection.java | 13 +++++++++---- .../hadoop/hbase/client/TestMetricsConnection.java | 8 +++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java index 9ba8dc100157..20d56328ed36 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java @@ -36,8 +36,8 @@ import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.ipc.RemoteWithExtrasException; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.ipc.RemoteException; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor; @@ -119,6 +119,7 @@ static String getScope(Configuration conf, String clusterId, Object connectionOb private static final String CNT_BASE = "rpcCount_"; private static final String FAILURE_CNT_BASE = "rpcFailureCount_"; + private static final String TOTAL_EXCEPTION_CNT = "rpcTotalExceptions"; private static final String LOCAL_EXCEPTION_CNT_BASE = "rpcLocalExceptions_"; private static final String REMOTE_EXCEPTION_CNT_BASE = "rpcRemoteExceptions_"; private static final String DRTN_BASE = "rpcCallDurationMs_"; @@ -651,9 +652,13 @@ public void updateRpc(MethodDescriptor method, Message param, CallStats stats, T getMetric(CNT_BASE + methodName, rpcCounters, counterFactory).inc(); if (e != null) { getMetric(FAILURE_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); - if (e instanceof RemoteWithExtrasException) { - getMetric(REMOTE_EXCEPTION_CNT_BASE + ((RemoteWithExtrasException) e).getClassName(), - rpcCounters, counterFactory).inc(); + getMetric(TOTAL_EXCEPTION_CNT, rpcCounters, counterFactory).inc(); + if (e instanceof RemoteException) { + String fullClassName = ((RemoteException) e).getClassName(); + String simpleClassName = (fullClassName != null) + ? fullClassName.substring(fullClassName.lastIndexOf(".") + 1) + : "unknown"; + getMetric(REMOTE_EXCEPTION_CNT_BASE + simpleClassName, rpcCounters, counterFactory).inc(); } else { getMetric(LOCAL_EXCEPTION_CNT_BASE + e.getClass().getSimpleName(), rpcCounters, counterFactory).inc(); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java index fff19805bb20..d70d2cf60006 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java @@ -155,7 +155,7 @@ public void testStaticMetrics() throws IOException { GetRequest.getDefaultInstance(), MetricsConnection.newCallStats(), null); METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Scan"), ScanRequest.getDefaultInstance(), MetricsConnection.newCallStats(), - new RemoteWithExtrasException("IOException", null, false, false)); + new RemoteWithExtrasException("java.io.IOException", null, false, false)); METRICS.updateRpc(ClientService.getDescriptor().findMethodByName("Multi"), MultiRequest.getDefaultInstance(), MetricsConnection.newCallStats(), new CallTimeoutException("test with CallTimeoutException")); @@ -217,6 +217,12 @@ public void testStaticMetrics() throws IOException { metricVal = (counter != null) ? counter.getCount() : 0; assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + // total exception + metricKey = "rpcTotalExceptions"; + counter = METRICS.getRpcCounters().get(metricKey); + metricVal = (counter != null) ? counter.getCount() : 0; + assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop * 2); + for (MetricsConnection.CallTracker t : new MetricsConnection.CallTracker[] { METRICS.getGetTracker(), METRICS.getScanTracker(), METRICS.getMultiTracker(), METRICS.getAppendTracker(), METRICS.getDeleteTracker(), METRICS.getIncrementTracker(),