From 2eb87e032582f3b398997f3877d6f27ec2b1653e Mon Sep 17 00:00:00 2001 From: Josh LEE Date: Thu, 25 Jan 2018 13:53:41 +0900 Subject: [PATCH] Update ApplicationMaster.scala I have one question. I think when maxNumExecutorFailures is calculated, MAX_EXECUTOR_FAILURES is already defined by specific by spark document (as numExecutors * 2, with minimum of 3) So the annotation added by me in the code is not valid. Give me the answer please. Thank you --- .../spark/deploy/yarn/ApplicationMaster.scala | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 4d5e3bb043671..4244faa8da6de 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -120,18 +120,19 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends // allocation is enabled), with a minimum of 3. private val maxNumExecutorFailures = { - val effectiveNumExecutors = - if (Utils.isDynamicAllocationEnabled(sparkConf)) { - sparkConf.get(DYN_ALLOCATION_MAX_EXECUTORS) - } else { - sparkConf.get(EXECUTOR_INSTANCES).getOrElse(0) - } - // By default, effectiveNumExecutors is Int.MaxValue if dynamic allocation is enabled. We need - // avoid the integer overflow here. - val defaultMaxNumExecutorFailures = math.max(3, - if (effectiveNumExecutors > Int.MaxValue / 2) Int.MaxValue else (2 * effectiveNumExecutors)) - - sparkConf.get(MAX_EXECUTOR_FAILURES).getOrElse(defaultMaxNumExecutorFailures) + // val effectiveNumExecutors = + // if (Utils.isDynamicAllocationEnabled(sparkConf)) { + // sparkConf.get(DYN_ALLOCATION_MAX_EXECUTORS) + // } else { + // sparkConf.get(EXECUTOR_INSTANCES).getOrElse(0) + // } + // // By default, effectiveNumExecutors is Int.MaxValue if dynamic allocation is enabled. We need + // // avoid the integer overflow here. + // val defaultMaxNumExecutorFailures = math.max(3, + // if (effectiveNumExecutors > Int.MaxValue / 2) Int.MaxValue else (2 * effectiveNumExecutors)) + + // sparkConf.get(MAX_EXECUTOR_FAILURES).getOrElse(defaultMaxNumExecutorFailures) + sparkConf.get(MAX_EXECUTOR_FAILURES) } @volatile private var exitCode = 0