-
Notifications
You must be signed in to change notification settings - Fork 28k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[SPARK-47488][k8s]fix driver pod stuck when driver on k8s #45667
base: master
Are you sure you want to change the base?
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -45,6 +45,7 @@ import org.apache.spark.internal.config.UI._ | |
import org.apache.spark.launcher.SparkLauncher | ||
import org.apache.spark.util._ | ||
import org.apache.spark.util.ArrayImplicits._ | ||
import org.apache.spark.util.SparkExitCode.EXIT_FAILURE | ||
|
||
/** | ||
* Whether to submit, kill, or request the status of an application. | ||
|
@@ -983,11 +984,19 @@ private[spark] class SparkSubmit extends Logging { | |
e | ||
} | ||
|
||
var DriverPodIsNormal: Boolean = if (args.master.startsWith("k8s")) true else false | ||
var driverThrow: Throwable = null | ||
try { | ||
app.start(childArgs.toArray, sparkConf) | ||
} catch { | ||
case t: Throwable => | ||
throw findCause(t) | ||
logWarning("Some ERR/Exception happened when app is running.") | ||
if (args.master.startsWith("k8s")) { | ||
DriverPodIsNormal = false | ||
driverThrow = t | ||
} else { | ||
throw findCause(t) | ||
} | ||
} finally { | ||
if (args.master.startsWith("k8s") && !isShell(args.primaryResource) && | ||
!isSqlShell(args.mainClass) && !isThriftServer(args.mainClass) && | ||
|
@@ -996,6 +1005,13 @@ private[spark] class SparkSubmit extends Logging { | |
SparkContext.getActive.foreach(_.stop()) | ||
} catch { | ||
case e: Throwable => logError(s"Failed to close SparkContext: $e") | ||
} finally { | ||
if (SparkContext.getActive.isEmpty) { | ||
if (!DriverPodIsNormal) { | ||
logError(s"Driver Pod will exit because: $driverThrow") | ||
System.exit(EXIT_FAILURE) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you provide the YARN code link for this case? Specifically,
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Here is yarn exit code. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What is the YARN exit code for your case? According to your PR description, it's not clear, @littlelittlewhite09 . It would be described in the PR description.
|
||
} | ||
} | ||
} | ||
} | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Did you want to use
DriverPodIsNormal
here?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes,
DriverPodIsNormal
applied here may be better.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Then, please use it.