HBASE-27845 Distinguish the mutate types of rpc error in MetricsConne…

…ction. (#5224)

Co-authored-by: fantasy <875282031@qq.com>
Co-authored-by: jay.zhu <jay.zhu@huolala.cn>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
(cherry picked from commit 047f077)

hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java

@@ -700,7 +700,28 @@ public void updateRpc(MethodDescriptor method, Message param, CallStats stats, T
       concurrentCallsPerServerHist.update(callsPerServer);
     }
     // Update the counter that tracks RPCs by type.
-    final String methodName = method.getService().getName() + "_" + method.getName();
+    StringBuilder methodName = new StringBuilder();
+    methodName.append(method.getService().getName()).append("_").append(method.getName());
+    // Distinguish mutate types.
+    if ("Mutate".equals(method.getName())) {
+      final MutationType type = ((MutateRequest) param).getMutation().getMutateType();
+      switch (type) {
+        case APPEND:
+          methodName.append("(Append)");
+          break;
+        case DELETE:
+          methodName.append("(Delete)");
+          break;
+        case INCREMENT:
+          methodName.append("(Increment)");
+          break;
+        case PUT:
+          methodName.append("(Put)");
+          break;
+        default:
+          methodName.append("(Unknown)");
+      }
+    }
     getMetric(CNT_BASE + methodName, rpcCounters, counterFactory).inc();
     if (e != null) {
       getMetric(FAILURE_CNT_BASE + methodName, rpcCounters, counterFactory).inc();
@@ -777,7 +798,7 @@ public void updateRpc(MethodDescriptor method, Message param, CallStats stats, T
       }
     }
     // Fallback to dynamic registry lookup for DDL methods.
-    updateRpcGeneric(methodName, stats);
+    updateRpcGeneric(methodName.toString(), stats);
   }
 
   public void incrCacheDroppingExceptions(Object exception) {

hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java

@@ -102,7 +102,7 @@ public void testMetricsConnectionScopeAsyncClient() throws IOException {
   }
 
   @Test
-  public void testMetricsWithMutiConnections() throws IOException {
+  public void testMetricsWithMultiConnections() throws IOException {
     Configuration conf = new Configuration();
     conf.setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true);
     conf.set(MetricsConnection.METRICS_SCOPE_KEY, "unit-test");
@@ -205,7 +205,8 @@ public void testStaticMetrics() throws IOException {
         MutateRequest.newBuilder()
           .setMutation(ProtobufUtil.toMutation(MutationType.PUT, new Put(foo))).setRegion(region)
           .build(),
-        MetricsConnection.newCallStats(), null);
+        MetricsConnection.newCallStats(),
+        new CallTimeoutException("test with CallTimeoutException"));
     }
 
     final String rpcCountPrefix = "rpcCount_" + ClientService.getDescriptor().getName() + "_";
@@ -215,40 +216,58 @@ public void testStaticMetrics() throws IOException {
     long metricVal;
     Counter counter;
 
-    for (String method : new String[] { "Get", "Scan", "Multi", "Mutate" }) {
+    for (String method : new String[] { "Get", "Scan", "Multi" }) {
       metricKey = rpcCountPrefix + method;
       metricVal = METRICS.getRpcCounters().get(metricKey).getCount();
-      assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop);
+      assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop);
 
       metricKey = rpcFailureCountPrefix + method;
       counter = METRICS.getRpcCounters().get(metricKey);
       metricVal = (counter != null) ? counter.getCount() : 0;
-      if (method.equals("Get") || method.equals("Mutate")) {
+      if (method.equals("Get")) {
         // no failure
-        assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0);
+        assertEquals("metric: " + metricKey + " val: " + metricVal, 0, metricVal);
       } else {
         // has failure
-        assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop);
+        assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop);
+      }
+    }
+
+    String method = "Mutate";
+    for (String mutationType : new String[] { "Append", "Delete", "Increment", "Put" }) {
+      metricKey = rpcCountPrefix + method + "(" + mutationType + ")";
+      metricVal = METRICS.getRpcCounters().get(metricKey).getCount();
+      assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop);
+
+      metricKey = rpcFailureCountPrefix + method + "(" + mutationType + ")";
+      counter = METRICS.getRpcCounters().get(metricKey);
+      metricVal = (counter != null) ? counter.getCount() : 0;
+      if (mutationType.equals("Put")) {
+        // has failure
+        assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop);
+      } else {
+        // no failure
+        assertEquals("metric: " + metricKey + " val: " + metricVal, 0, metricVal);
       }
     }
 
     // remote exception
     metricKey = "rpcRemoteExceptions_IOException";
     counter = METRICS.getRpcCounters().get(metricKey);
     metricVal = (counter != null) ? counter.getCount() : 0;
-    assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop);
+    assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop);
 
     // local exception
     metricKey = "rpcLocalExceptions_CallTimeoutException";
     counter = METRICS.getRpcCounters().get(metricKey);
     metricVal = (counter != null) ? counter.getCount() : 0;
-    assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop);
+    assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop * 2);
 
     // total exception
     metricKey = "rpcTotalExceptions";
     counter = METRICS.getRpcCounters().get(metricKey);
     metricVal = (counter != null) ? counter.getCount() : 0;
-    assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop * 2);
+    assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop * 3);
 
     for (MetricsConnection.CallTracker t : new MetricsConnection.CallTracker[] {
       METRICS.getGetTracker(), METRICS.getScanTracker(), METRICS.getMultiTracker(),