From 8325e8a3c7d156454e784cc852d7ed9600161cf7 Mon Sep 17 00:00:00 2001 From: Cheolsoo Park Date: Thu, 2 Apr 2015 23:35:24 -0700 Subject: [PATCH 1/3] Make it possible to kill AM in YARN cluster mode when the client is terminated --- docs/running-on-yarn.md | 8 ++++++++ .../org/apache/spark/deploy/yarn/Client.scala | 18 +++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 853c9f26b0ec..d33832cf166b 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -46,6 +46,14 @@ Most of the configs are the same for Spark on YARN as for other deployment modes In cluster mode, use spark.driver.cores instead. + + spark.yarn.am.force.shutdown + false + + In yarn-cluster mode, the YARN Application Master is not killed when the client is terminated by default. + But if spark.yarn.am.force.shutdown is set true, the YARN AM is force shutdown. + + spark.yarn.am.waitTime 100s diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 1091ff54b046..1318bec0b25d 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -17,7 +17,7 @@ package org.apache.spark.deploy.yarn -import java.net.{InetAddress, UnknownHostException, URI, URISyntaxException} +import java.net.{URI, InetAddress, UnknownHostException} import java.nio.ByteBuffer import scala.collection.JavaConversions._ @@ -26,7 +26,6 @@ import scala.reflect.runtime.universe import scala.util.{Try, Success, Failure} import com.google.common.base.Objects - import org.apache.hadoop.io.DataOutputBuffer import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier @@ -37,7 +36,7 @@ import org.apache.hadoop.mapred.Master import org.apache.hadoop.mapreduce.MRJobConfig import org.apache.hadoop.security.{Credentials, UserGroupInformation} import org.apache.hadoop.security.token.Token -import org.apache.hadoop.util.StringUtils +import org.apache.hadoop.util.{ShutdownHookManager, StringUtils} import org.apache.hadoop.yarn.api._ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment import org.apache.hadoop.yarn.api.protocolrecords._ @@ -46,8 +45,7 @@ import org.apache.hadoop.yarn.client.api.{YarnClient, YarnClientApplication} import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException import org.apache.hadoop.yarn.util.Records - -import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext, SparkException} +import org.apache.spark.{SparkConf, SparkContext, SecurityManager, SparkException, Logging} import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.util.Utils @@ -112,6 +110,16 @@ private[spark] class Client( // Finally, submit and monitor the application logInfo(s"Submitting application ${appId.getId} to ResourceManager") yarnClient.submitApplication(appContext) + + // In YARN Cluster mode, AM is not killed when the client is terminated by default. + // But if spark.yarn.force.shutdown.am is set true, AM is force shutdown. + if (isClusterMode && sparkConf.getBoolean("spark.yarn.am.force.shutdown", false)) { + val shutdownHook = new Runnable { + override def run() { yarnClient.killApplication(appId) } + } + ShutdownHookManager.get().addShutdownHook(shutdownHook, 0) + } + appId } From 6de99db9de9ddb0e6d1fd3da07816815718d76db Mon Sep 17 00:00:00 2001 From: Cheolsoo Park Date: Fri, 3 Apr 2015 14:20:19 -0700 Subject: [PATCH 2/3] Kill AM only if it's in ACCEPTED and RUNNING states. Don't kill when AM is in FINISHED and FAILED states. --- .../org/apache/spark/deploy/yarn/Client.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 1318bec0b25d..4f6e65891361 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -17,7 +17,7 @@ package org.apache.spark.deploy.yarn -import java.net.{URI, InetAddress, UnknownHostException} +import java.net.{InetAddress, UnknownHostException, URI, URISyntaxException} import java.nio.ByteBuffer import scala.collection.JavaConversions._ @@ -26,6 +26,7 @@ import scala.reflect.runtime.universe import scala.util.{Try, Success, Failure} import com.google.common.base.Objects + import org.apache.hadoop.io.DataOutputBuffer import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier @@ -45,7 +46,8 @@ import org.apache.hadoop.yarn.client.api.{YarnClient, YarnClientApplication} import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException import org.apache.hadoop.yarn.util.Records -import org.apache.spark.{SparkConf, SparkContext, SecurityManager, SparkException, Logging} + +import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext, SparkException} import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.util.Utils @@ -112,10 +114,16 @@ private[spark] class Client( yarnClient.submitApplication(appContext) // In YARN Cluster mode, AM is not killed when the client is terminated by default. - // But if spark.yarn.force.shutdown.am is set true, AM is force shutdown. + // But if spark.yarn.am.force.shutdown is set true, AM is force shutdown. if (isClusterMode && sparkConf.getBoolean("spark.yarn.am.force.shutdown", false)) { val shutdownHook = new Runnable { - override def run() { yarnClient.killApplication(appId) } + override def run() { + val report = getApplicationReport(appId) + if (report.getYarnApplicationState == YarnApplicationState.ACCEPTED || + report.getYarnApplicationState == YarnApplicationState.RUNNING) { + yarnClient.killApplication(appId) + } + } } ShutdownHookManager.get().addShutdownHook(shutdownHook, 0) } From 2a3fa381708ce5319ca3786a079c866b70467e81 Mon Sep 17 00:00:00 2001 From: Cheolsoo Park Date: Tue, 14 Apr 2015 20:44:27 -0700 Subject: [PATCH 3/3] Minor rewording in docs --- docs/running-on-yarn.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index d33832cf166b..da5d7386944e 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -50,8 +50,8 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.am.force.shutdown false - In yarn-cluster mode, the YARN Application Master is not killed when the client is terminated by default. - But if spark.yarn.am.force.shutdown is set true, the YARN AM is force shutdown. + In yarn-cluster mode, the YARN Application Master is not killed by default even when the client is terminated. + But if spark.yarn.am.force.shutdown is set true, the YARN AM is force shutdown when the client exits.