HDDS-4027. Suppress ERROR message when SCM attempt to create addition…

…al pipelines. (#1265)
apache · Jul 29, 2020 · 0892fab · 0892fab
1 parent a96553e
commit 0892fab
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 2 deletions.
diff --git a/...hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/...hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
 import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -56,6 +57,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE;
+
 /**
  * Implements api needed for management of pipelines. All the write operations
  * for pipelines must come via PipelineManager. It synchronises all write
@@ -272,8 +275,15 @@ public Pipeline createPipeline(ReplicationType type,
       recordMetricsForPipeline(pipeline);
       return pipeline;
     } catch (IOException ex) {
-      LOG.error("Failed to create pipeline of type {} and factor {}. " +
-          "Exception: {}", type, factor, ex.getMessage());
+      if (ex instanceof SCMException &&
+          ((SCMException) ex).getResult() == FAILED_TO_FIND_SUITABLE_NODE) {
+        // Avoid spam SCM log with errors when SCM has enough open pipelines
+        LOG.debug("Can't create more pipelines of type {} and factor {}. " +
+            "Reason: {}", type, factor, ex.getMessage());
+      } else {
+        LOG.error("Failed to create pipeline of type {} and factor {}. " +
+            "Exception: {}", type, factor, ex.getMessage());
+      }
       metrics.incNumPipelineCreationFailed();
       throw ex;
     } finally {

diff --git a/.../server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/.../server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
@@ -76,6 +76,7 @@
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static org.slf4j.event.Level.INFO;
 
 /**
  * Test cases to verify PipelineManager.
@@ -304,6 +305,8 @@ public void testPipelineCreationFailedMetric() throws Exception {
         "NumPipelineCreationFailed", metrics);
     Assert.assertEquals(0, numPipelineCreateFailed);
 
+    LogCapturer logs = LogCapturer.captureLogs(SCMPipelineManager.getLog());
+    GenericTestUtils.setLogLevel(SCMPipelineManager.getLog(), INFO);
     //This should fail...
     try {
       pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS,
@@ -313,6 +316,10 @@ public void testPipelineCreationFailedMetric() throws Exception {
       // pipeline creation failed this time.
       Assert.assertEquals(SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE,
           ioe.getResult());
+      Assert.assertFalse(logs.getOutput().contains(
+          "Failed to create pipeline of type"));
+    } finally {
+      logs.stopCapturing();
     }
 
     metrics = getMetrics(