java -> scala makes it more simple

apache · Nov 16, 2018 · 8ab6c3b · 8ab6c3b
1 parent 965a984
commit 8ab6c3b
Showing 1 changed file with 13 additions and 17 deletions.
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -20,7 +20,6 @@ package org.apache.spark.deploy.yarn
 import java.util.Collections
 import java.util.concurrent._
 import java.util.concurrent.atomic.AtomicInteger
-import java.util.regex.Pattern
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
@@ -598,23 +597,22 @@ private[yarn] class YarnAllocator(
             (false, s"Container ${containerId}${onHostStr} was preempted.")
           // Should probably still count memory exceeded exit codes towards task failures
           case VMEM_EXCEEDED_EXIT_CODE =>
-            val suggestion = if (conf.getBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED,
+            val diag = VMEM_EXCEEDED_PATTERN.findFirstIn(completedContainer.getDiagnostics)
+              .map(_.concat(".")).getOrElse("")
+            val additional = if (conf.getBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED,
               YarnConfiguration.DEFAULT_NM_VMEM_CHECK_ENABLED)) {
-              s"Consider disabling ${YarnConfiguration.NM_VMEM_CHECK_ENABLED} because of YARN-4714"
+              s"or disabling ${YarnConfiguration.NM_VMEM_CHECK_ENABLED} because of YARN-4714"
             } else {
-              ""
+              s"or boosting ${YarnConfiguration.NM_VMEM_PMEM_RATIO}"
             }
-            val matcher = VMEM_EXCEEDED_PATTERN.matcher(completedContainer.getDiagnostics)
-            val diag = if (matcher.find()) " " + matcher.group() + "." else ""
-            val message =
-              s"Container killed by YARN for exceeding virtual memory limits.$diag $suggestion."
+            val message = "Container killed by YARN for exceeding virtual memory limits." +
+              s" $diag Consider boosting ${EXECUTOR_MEMORY_OVERHEAD.key} $additional."
             (true, message)
           case PMEM_EXCEEDED_EXIT_CODE =>
-            val suggestion = s"Consider boosting ${EXECUTOR_MEMORY_OVERHEAD.key}"
-            val matcher = PMEM_EXCEEDED_PATTERN.matcher(completedContainer.getDiagnostics)
-            val diag = if (matcher.find()) " " + matcher.group() + "." else ""
-            val message =
-              s"Container killed by YARN for exceeding physical memory limits.$diag $suggestion."
+            val diag = PMEM_EXCEEDED_PATTERN.findFirstIn(completedContainer.getDiagnostics)
+              .map(_.concat(".")).getOrElse("")
+            val message = "Container killed by YARN for exceeding physical memory limits." +
+              s" $diag Consider boosting ${EXECUTOR_MEMORY_OVERHEAD.key}."
             (true, message)
           case _ =>
             // all the failures which not covered above, like:
@@ -746,10 +744,8 @@ private[yarn] class YarnAllocator(
 
 private object YarnAllocator {
   val MEM_REGEX = "[0-9.]+ [KMG]B"
-  val PMEM_EXCEEDED_PATTERN =
-    Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used")
-  val VMEM_EXCEEDED_PATTERN =
-    Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used")
+  val PMEM_EXCEEDED_PATTERN = raw"$MEM_REGEX of $MEM_REGEX physical memory used".r
+  val VMEM_EXCEEDED_PATTERN = raw"$MEM_REGEX of $MEM_REGEX virtual memory used".r
   val VMEM_EXCEEDED_EXIT_CODE = -103
   val PMEM_EXCEEDED_EXIT_CODE = -104
 }