From 86818817dcdf348da2dee5f9eef44c34f25a71e6 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Thu, 9 Oct 2014 16:18:34 +0800 Subject: [PATCH 1/2] [SPARK-3877] Throw an exception when application is not successful so that the exit code wil be set to 1 --- .../org/apache/spark/deploy/yarn/Client.scala | 14 +++++++++----- .../org/apache/spark/deploy/yarn/Client.scala | 14 +++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 3607eed1f1e2b..e37669e7089b9 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{Apps, Records} -import org.apache.spark.{Logging, SparkConf} +import org.apache.spark.{Logging, SparkConf, SparkException} /** * Version of [[org.apache.spark.deploy.yarn.ClientBase]] tailored to YARN's alpha API. @@ -84,7 +84,9 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa def run() { val appId = runApp() - monitorApplication(appId) + if (!monitorApplication(appId)) { + throw new SparkException("Application is not successful") + } } def logClusterResourceDetails() { @@ -138,10 +140,12 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa ) val state = report.getYarnApplicationState() - if (state == YarnApplicationState.FINISHED || - state == YarnApplicationState.FAILED || + if (state == YarnApplicationState.FINISHED) { + return report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED + } + if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) { - return true + return false } } true diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 0b55282b75598..ae914a8ebeb40 100644 --- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.Records -import org.apache.spark.{Logging, SparkConf} +import org.apache.spark.{Logging, SparkConf, SparkException} /** @@ -94,7 +94,9 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa def run() { val appId = runApp() - monitorApplication(appId) + if (!monitorApplication(appId)) { + throw new SparkException("Application is not successful") + } } def logClusterResourceDetails() { @@ -144,10 +146,12 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa ) val state = report.getYarnApplicationState() - if (state == YarnApplicationState.FINISHED || - state == YarnApplicationState.FAILED || + if (state == YarnApplicationState.FINISHED) { + return report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED + } + if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) { - return true + return false } } true From 370198425dce1d2d268b00438366b3e3b56a8743 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Fri, 17 Oct 2014 10:50:14 +0800 Subject: [PATCH 2/2] Remove System.exit from Client.scala --- .../scala/org/apache/spark/deploy/yarn/Client.scala | 13 ++----------- .../scala/org/apache/spark/deploy/yarn/Client.scala | 13 ++----------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index e37669e7089b9..c51ce62c1160e 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -166,16 +166,7 @@ object Client { val sparkConf = new SparkConf - try { - val args = new ClientArguments(argStrings, sparkConf) - new Client(args, sparkConf).run() - } catch { - case e: Exception => { - Console.err.println(e.getMessage) - System.exit(1) - } - } - - System.exit(0) + val args = new ClientArguments(argStrings, sparkConf) + new Client(args, sparkConf).run() } } diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index ae914a8ebeb40..da05f7ddc5cb9 100644 --- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -172,17 +172,8 @@ object Client { System.setProperty("SPARK_YARN_MODE", "true") val sparkConf = new SparkConf() - try { - val args = new ClientArguments(argStrings, sparkConf) - new Client(args, sparkConf).run() - } catch { - case e: Exception => { - Console.err.println(e.getMessage) - System.exit(1) - } - } - - System.exit(0) + val args = new ClientArguments(argStrings, sparkConf) + new Client(args, sparkConf).run() } }